Kaggle competition: Digit Recognizer
( image_train_mnist, label_train_mnist),(image_test_mnist, label_test_mnist)= mnist.load_data()
image_mnist = np.concatenate(( image_train_mnist, image_test_mnist))
label_mnist = np.concatenate(( label_train_mnist, label_test_mnist))
image_mnist = image_mnist.reshape(-1,28,28,1)
 image_mnist = image_mnist.astype(np.float32)/ 255
label_mnist = tf.keras.utils.to_categorical(label_mnist,num_classes=10)
 images = np.concatenate(( train_image, image_mnist))
labels = np.concatenate(( train_label, label_mnist))
print("training image dataset shape:", images.shape)
 print("training label dataset shape:", labels.shape)
 show_images(images[:25], labels[:25], shape=(5,5))<define_variables>
data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv' )
datagen = tf.keras.preprocessing.image.ImageDataGenerator(
 rotation_range=20,
width_shift_range=0.20,
shear_range=15,
zoom_range=0.10,
validation_split=0.25,
horizontal_flip=False 
)
 train_generator = datagen.flow(
 images,
labels,
batch_size=256,
subset='training', 
)
 validation_generator = datagen.flow(
 images,
labels,
batch_size=64,
subset='validation',
)<choose_model_class>
MinMaxScaler = preprocessing.MinMaxScaler()
input_data = MinMaxScaler.fit_transform(input_data )
def create_model() :
model = tf.keras.Sequential([
tf.keras.layers.Reshape(( 28, 28, 1)) ,
tf.keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", padding="same", input_shape=(28,28,1)) ,
tf.keras.layers.MaxPool2D(( 2,2)) ,
tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu", padding="same"),
tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu", padding="same"),
tf.keras.layers.MaxPool2D(( 2,2)) ,
tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), activation="relu", padding="same"),
tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), activation="relu", padding="same"),
tf.keras.layers.MaxPool2D(( 2,2)) ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(512, activation="sigmoid"),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(512, activation="sigmoid"),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(256, activation="sigmoid"),
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(10, activation="sigmoid")
 ])
 model.compile(
 optimizer="adam", loss = 'categorical_crossentropy', metrics = ['accuracy'] 
)
 return model
model = create_model()<train_model>
test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
history = model.fit_generator(train_generator, epochs=60, validation_data=validation_generator, callbacks=[reduce_lr,checkpoint], verbose=1 )<save_to_csv>
test_data = MinMaxScaler.fit_transform(test)
 test_data = test_data.reshape(-1, 28, 28, 1)
 test_data.shape
df = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" ).astype("float32")/ 255.0
res = tf.keras.backend.argmax(model.predict(df))
csv = pd.DataFrame({'ImageId': range(1, len(res)+ 1), "Label": res})
 csv.to_csv('submission.csv', index=False )<import_modules>
X_train, X_test, y_train, y_test = train_test_split(input_data, output_data.values, test_size=0.3, random_state=1 )
import numpy as np
import pandas as pd
import scipy
import sklearn
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from itertools import product
from sklearn.cluster import KMeans
import lightgbm as lgb<define_variables>
from keras.utils.np_utils import to_categorical
dpath = '.. /input/competitive-data-science-predict-future-sales/'
adpath ='.. /input/predict-future-sales/'<load_from_csv>
y_train_cat = to_categorical(y_train, 10)
 y_test_cat = to_categorical(y_test, 10 )
df_train = pd.read_csv(dpath + 'sales_train.csv')
 df_test = pd.read_csv(dpath + 'test.csv', index_col='ID')
 df_shops = pd.read_csv(dpath + 'shops.csv', index_col='shop_id')
 df_items = pd.read_csv(dpath + 'items.csv', index_col='item_id')
 df_itemcat = pd.read_csv(dpath + 'item_categories.csv', index_col='item_category_id')
 sample_submission = pd.read_csv(dpath + 'sample_submission.csv', index_col='ID' )<load_from_csv>
from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout, AveragePooling2D
import keras.backend as K
calendar = pd.read_csv(adpath + 'calendar.csv', dtype='int16' )<categorify>
generator = ImageDataGenerator(
 width_shift_range=0.1,
height_shift_range=0.1,
rotation_range = 20,
shear_range = 0.3,
zoom_range = 0.3,
horizontal_flip = True)
 generator.fit(X_train )
def shop_name2city(sn):
sn = sn.split() [0]
if sn == 'Цифровой' or sn == 'Интернет-магазин': sn = 'Internet'
if sn[0] == '!': sn = sn[1:]
return sn
df_shops['city'] = df_shops['shop_name'].apply(shop_name2city)
 df_shops['city_enc'] = LabelEncoder().fit_transform(df_shops['city'] ).astype('int8')
 city_info = pd.read_pickle(adpath + 'city_info.pkl')
 df_shops['city_size'] = df_shops['city'].map(city_info['city_size'] )<categorify>
K.clear_session()
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='tanh', input_shape=(28, 28, 1), padding="SAME"))
model.add(Conv2D(32,(3, 3), activation='tanh', padding="SAME"))
model.add(AveragePooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(64,(3, 3), activation='tanh', padding="SAME"))
model.add(Conv2D(64,(3, 3), activation='tanh', padding="SAME"))
model.add(AveragePooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(128,(3, 3), activation='tanh', padding="SAME"))
model.add(Conv2D(128,(3, 3), activation='tanh', padding="SAME"))
model.add(AveragePooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(784, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(256, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='Adamax',
metrics=['accuracy'] )
class Items() :
def __init__(self, df_items, df_itemcat):
self.df_items = df_items
self.df_itemcat = df_itemcat
self.set_hl_cat()
self.make_items_ext()
self.item_features = ['item_category_id', 'hl_cat_id']
def set_hl_cat(self):
self.df_itemcat['hl_cat_id'] = self.df_itemcat['item_category_name'].str.split(n=1, expand=True)[0]
self.df_itemcat['hl_cat_id'] = LabelEncoder().fit_transform(self.df_itemcat['hl_cat_id'])
 def make_items_ext(self):
self.df_items = df_items.merge(self.df_itemcat, how = 'left',
left_on = 'item_category_id', right_index = True)
 def get_items_df(self):
return self.df_items[self.item_features].astype('int32' )<concatenate>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
items = Items(df_items, df_itemcat )<prepare_output>
K.clear_session()
hh = model.fit(generator.flow(X_train, y_train_cat, batch_size=64), validation_data=(X_test, y_test_cat),
steps_per_epoch=len(X_train)/ 64, epochs=30, verbose=1, callbacks=[learning_rate_reduction] )
class TT_Extended() :
def __init__(self, df_train, df_test, items, df_shops, calendar, cmode, verbose=True):
self.info = verbose
self.df_train = df_train.copy()
self.df_test = df_test.copy()
self.df_shops = df_shops.copy()
self.calendar = self.set_calender(calendar.copy())
 self.idx_columns = ['date_block_num', 'shop_id', 'item_id']
self.df_test['date_block_num'] = 34
self.df_test['item_cnt_month'] = 0.
 self.df_test['item_cnt_month'] = self.df_test['item_cnt_month'].astype('float32')
 self.df_train[self.idx_columns] = self.df_train[self.idx_columns].astype('int32')
 self.df_test[self.idx_columns] = self.df_test[self.idx_columns].astype('int32')
 self.df_train_cleaning(cmode)
 self.item_mean_features = []
self.shop_mean_features = []
self.lag_names_to_clip = []
self.df_items = items.get_items_df()
self.item_ext_features = list(self.df_items.columns)
 self.df_items_ext = self.items_ext()
self.df_bb = self.build_bb()
self.df_train_ext = self.df_train_agg(cmin = 0, cmax = 1000, drop = None)
 self.add_test_df()
self.df_train_ext = self.df_train_extention()
def df_train_cleaning(self, mode):
assert mode in ['keep', 'drop', 'block', 'total', 't+b']
if self.info: print('Cleaning train dataframe...( Mode -', mode, ')')
 shop_idx = self.df_train[(self.df_train['shop_id'] == 9)| 
(self.df_train['shop_id'] == 20)].index
self.df_train.drop(shop_idx, inplace=True)
 self.df_train = self.df_train[(self.df_train['item_price'] > 0)& 
(self.df_train['item_price'] < 51000)]
self.df_train = self.df_train[self.df_train['item_cnt_day'] <= 1000]
shop_repl_dict = {0 : 57, 1 : 58, 11 : 10, 40 : 39}
self.df_train['shop_id'] = self.df_train['shop_id'].apply(
 lambda s: shop_repl_dict.get(s)if s in shop_repl_dict else s)
 if mode == 'drop':
self.df_train = self.df_train[self.df_train['item_cnt_day'] > 0]
elif mode == 'block':
item_block_cnt = self.df_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_block_cnt[item_block_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['date_block_num', 'shop_id', 'item_id'] ).index.isin(items_to_drop)]
elif mode == 'total':
item_total_cnt = self.df_train.groupby(['shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_total_cnt[item_total_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['shop_id', 'item_id'] ).index.isin(items_to_drop)]
elif mode == 't+b':
item_total_cnt = self.df_train.groupby(['shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_total_cnt[item_total_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['shop_id', 'item_id'] ).index.isin(items_to_drop)]
item_block_cnt = self.df_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_block_cnt[item_block_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['date_block_num', 'shop_id', 'item_id'] ).index.isin(items_to_drop)]
return
def set_calender(self, calendar):
calendar['date_block_num'] =(calendar['year'] - 2013)*12 +(calendar['month'] - 1)
 calendar['hdays'] = calendar['mdays'] - calendar['wdays']
calendar.set_index('date_block_num', inplace=True)
 return calendar
def items_ext(self):
dfi = self.df_items.copy()
dfi['fsb'] = self.df_train.groupby('item_id')['date_block_num'].min()
dfi['fsb'].fillna(34, inplace=True)
 dfi['fsb'] = dfi['fsb'].astype('int8')
 self.item_ext_features += ['fsb']
return dfi
def build_bb(self):
if self.info: print('Building index dataframe...')
 df_work = []
for block_num in self.df_train['date_block_num'].unique() :
cur_shops = self.df_train.loc[self.df_train['date_block_num'] == block_num, 'shop_id'].unique()
cur_items = self.df_train.loc[self.df_train['date_block_num'] == block_num, 'item_id'].unique()
df_work.append(np.array(list(product(*[[block_num], cur_shops, cur_items])) , dtype='int32'))
df_work = pd.DataFrame(np.vstack(df_work), columns = self.idx_columns)
 return df_work
def df_train_agg(self, cmin = 0, cmax = 20, drop = None):
if self.info: print('Aggregation...')
 df_work = self.df_train.groupby(self.idx_columns ).agg({'item_price' : np.mean,
'item_cnt_day': np.sum})
 df_work.reset_index(inplace=True)
 df_work = df_work.rename(columns={'item_cnt_day': 'item_cnt_month'})
 df_work = pd.merge(self.df_bb, df_work, on=self.idx_columns, how='left')
 df_work['item_cnt_month'] = df_work['item_cnt_month'].astype('float32' ).fillna(0.).clip(cmin, cmax)
 df_work['item_price'] = df_work['item_price'].astype('float32' ).fillna(0.)
df_tmp = self.df_train[self.df_train['item_cnt_day'] > 0].groupby(self.idx_columns ).agg({'item_cnt_day': 'count'})
 df_tmp.reset_index(inplace=True)
 df_tmp = df_tmp.rename(columns={'item_cnt_day': 'item_rate_month'})
 df_work = pd.merge(df_work, df_tmp, on=self.idx_columns, how='left')
 df_work['item_rate_month'] = df_work['item_rate_month'].astype('float32' ).fillna(0.)
del df_tmp
if drop: df_work.drop(drop, axis=1, inplace=True)
 return df_work
def add_test_df(self):
self.df_train_ext = pd.concat([self.df_train_ext, self.df_test], ignore_index=True,
sort=False, keys=self.idx_columns)
 def add_item_means(self, df, feature = None):
if feature == None :
group_items = ['date_block_num','item_id']
feature = 'item_cnt'
else:
group_items = ['date_block_num','item_id'] + [feature]
feature_mean_name = feature + '_mean'
if self.info: print('Adding item means for', feature, '...')
 df_tmp = df.groupby(group_items)['item_cnt_month'].mean()
df_tmp = df_tmp.reset_index().rename(columns = {'item_cnt_month': feature_mean_name})
 df = pd.merge(df, df_tmp, on=group_items, how='left')
 self.item_mean_features.append(feature_mean_name)
 del df_tmp
return df
def add_shop_means(self, df, feature):
group_items = ['date_block_num', 'shop_id'] + [feature]
feature_mean_name = feature + '_mean'
if self.info: print('Adding shop means for', feature, '...')
 df_tmp = df.groupby(group_items)['item_cnt_month'].mean()
df_tmp = df_tmp.reset_index().rename(columns = {'item_cnt_month': feature_mean_name})
 df = pd.merge(df, df_tmp, on=group_items, how='left')
 self.shop_mean_features.append(feature)
 del df_tmp
return df
def df_train_extention(self, test_cat_only = False):
df_work = self.df_train_ext.merge(df_shops[['city_enc', 'city_size']], how = 'left', on = 'shop_id')
 df_work = df_work.merge(self.calendar[['mdays', 'wdays', 'hdays']],
how = 'left', left_on = 'date_block_num', right_index = True)
 df_work = df_work.merge(self.df_items_ext[self.item_ext_features],
how = 'left', left_on = 'item_id', right_index = True)
 ssbn = self.df_train.groupby('shop_id')['date_block_num'].min().astype('int8')
 ssbn.name = 'ssbn'
df_work = df_work.merge(ssbn, how = 'left', on = 'shop_id')
 if test_cat_only:
test_cat = df_work[df_work['date_block_num'] == 34]['item_category_id'].unique()
df_work = df_work[df_work['item_category_id'].isin(test_cat)]
return df_work
def make_base_df(self, keepnans = False):
df_work = self.df_train_ext.copy()
if keepnans:
fill_value = None
else:
fill_value = 0
df_work = pd.pivot_table(df_work, values='item_cnt_month', index=['shop_id', 'item_id'],
columns = 'date_block_num', aggfunc=np.sum, fill_value = fill_value)
 df_work.columns.name = ''
return df_work
def add_total_cnt(self, df):
if self.info: print('Adding total count...')
 df_base = self.make_base_df()
for i in range(1, 33):
df_base[i + 1] += df_base[i]
df_base = df_base.shift(1, axis=1 ).loc[:, 1:].astype('int32')
 df_base = df_base.melt(var_name='date_block_num', value_name='total_cnt', ignore_index=False)
 df = df.merge(df_base, how='left', on=self.idx_columns)
 del df_base
return df
def add_item_lags(self, df, feature_name, nlags=3, keepnans=False, dnc=False):
if self.info: print('Adding item lags for', feature_name, '...')
 df_tmp = df[['date_block_num', 'shop_id', 'item_id', feature_name]]
for i in range(nlags, 0, -1):
lag_feature_name = feature_name +'_lag-' + str(i)
 if not dnc: self.lag_names_to_clip.append(lag_feature_name)
 df_shifted = df_tmp.copy()
df_shifted.columns = ['date_block_num', 'shop_id', 'item_id', lag_feature_name]
df_shifted['date_block_num'] += i
df = pd.merge(df, df_shifted, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 if keepnans:
df[lag_feature_name] = df[lag_feature_name].astype('float32')
 else:
df[lag_feature_name] = df[lag_feature_name].fillna(0 ).astype('float32')
 del df_tmp
return df
def add_shop_lags(self, df, feature_name, nlags=3, dnc=False):
mean_feature_name = feature_name + '_mean'
if self.info: print('Adding lags for', mean_feature_name, '...')
 df_tmp = df[['date_block_num', 'shop_id', feature_name, mean_feature_name]]
for i in range(nlags, 0, -1):
lag_feature_name = mean_feature_name + '_lag-' + str(i)
 if not dnc: self.lag_names_to_clip.append(lag_feature_name)
 df_shifted = df_tmp.copy()
df_shifted.columns = ['date_block_num', 'shop_id', feature_name, lag_feature_name]
df_shifted['date_block_num'] += i
df = pd.merge(df, df_shifted.drop_duplicates() , on=['date_block_num', 'shop_id', feature_name], how='left')
 df[lag_feature_name] = df[lag_feature_name].fillna(0 ).astype('float32')
 del df_tmp
del df_shifted
return df
def shop_clustering(self, df_work):
print('No shop clusters provided')
 return df_work
def build_work_db(self, hd, item_mean_features = [], shop_mean_features = [], add_total_cnt = False):
if self.info: print('Building work dataframe...')
 df_work = self.df_train_ext.copy()
df_work = self.add_item_means(df_work)
 for mf in item_mean_features:
df_work = self.add_item_means(df_work, mf)
 for mf in shop_mean_features:
df_work = self.add_shop_means(df_work, mf)
 df_work = self.add_item_lags(df_work, 'item_cnt_month', hd, keepnans=False)
 df_work = self.add_item_lags(df_work, 'item_rate_month', hd, dnc=True)
 df_work = self.add_item_lags(df_work, 'item_price', hd, dnc=True)
 for mf in self.item_mean_features:
df_work = self.add_item_lags(df_work, mf, hd)
 for mf in self.shop_mean_features:
df_work = self.add_shop_lags(df_work, mf, hd)
 df_work.drop(df_work[df_work['date_block_num'] < hd].index, inplace=True)
 df_work.drop(self.item_mean_features, axis=1, inplace=True)
 df_work.drop(['item_category_id_mean', 'item_price', 'item_rate_month'], axis=1, inplace=True)
 self.item_mean_features = []
self.shop_mean_features = []
df_work['qmean'] = df_work[['item_cnt_month_lag-1',
'item_cnt_month_lag-2',
'item_cnt_month_lag-3']].mean(skipna=True, axis=1)
 df_work['new'] = df_work['fsb'] == df_work['date_block_num']
df_work['fsb'] = df_work['date_block_num'] - df_work['fsb']
df_work['ssbn'] = df_work['date_block_num'] - df_work['ssbn']
df_work['month'] =(df_work['date_block_num']%12 ).astype('int8')
 idx = df_work[(df_work['item_price_lag-1'] == 0)&(df_work['item_price_lag-2'] != 0)].index
df_work.loc[idx, 'item_price_lag-1'] = df_work.loc[idx, 'item_price_lag-2']
idx = df_work[(df_work['item_price_lag-2'] == 0)&(df_work['item_price_lag-3'] != 0)].index
df_work.loc[idx, 'item_price_lag-2'] = df_work.loc[idx, 'item_price_lag-3']
df_work['grad-1'] = df_work['item_cnt_month_lag-1']/df_work['item_cnt_month_lag-2']
df_work['grad-1'] = df_work['grad-1'].replace([np.inf, -np.inf], np.nan ).fillna(0.)
df_work['grad-2'] = df_work['item_cnt_month_lag-2']/df_work['item_cnt_month_lag-3']
df_work['grad-2'] = df_work['grad-2'].replace([np.inf, -np.inf], np.nan ).fillna(0.)
df_work = self.shop_clustering(df_work)
 if add_total_cnt: df_work = self.add_total_cnt(df_work)
 col2clip = ['item_cnt_month', 'qmean'] + self.lag_names_to_clip
df_work[col2clip] = df_work[col2clip].clip(0, 20)
 return df_work
def get_work_db(self, hd = 3, item_mean_features = [],
shop_mean_features = [],
drop_features = None,
add_total_cnt = False):
df_work = self.build_work_db(hd, item_mean_features, shop_mean_features, add_total_cnt)
 if drop_features == None:
return df_work
else:
return df_work.drop(drop_features, axis = 1 )<concatenate>
trained_weights = model.get_weights()
%%time
pfs = TT_Extended(df_train, df_test, items, df_shops, calendar, cmode='total' )<set_options>
learning_rate_reduction_2 = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.3,
min_lr=0.00001 )
pfs.shop_clustering = simple_shop_clustering<drop_column>
K.clear_session()
model.set_weights(trained_weights)
 h = model.fit(X_train, y_train_cat,
batch_size = 64,
validation_data=(X_test, y_test_cat),
epochs=50,
verbose=1, callbacks=[learning_rate_reduction_2] )
%%time
df_work = pfs.get_work_db(hd = 3,
item_mean_features = ['city_enc'],
shop_mean_features = ['item_category_id'],
drop_features = ['wdays', 'hdays', 'ssbn'],
add_total_cnt = False ).copy()<feature_engineering>
model.evaluate(X_test, y_test_cat )
df_work['city_size'] = df_work['city_size'].round(1 )<prepare_x_and_y>
predictions = model.predict(test_data )
X_train = df_work[df_work.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = df_work[df_work.date_block_num < 33]['item_cnt_month']
X_valid = df_work[df_work.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_valid = df_work[df_work.date_block_num == 33]['item_cnt_month']
X_test = df_work[df_work.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<drop_column>
pred = np.argmax(predictions, axis = 1)
 pred.shape
del df_work<init_hyperparams>
sample = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )
<save_to_csv><EOS> 
output = pd.DataFrame({'ImageId': sample.ImageId, 'Label': pred})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )