Kaggle competition: Digit Recognizer
shops['shop_category'] = LabelEncoder().fit_transform(shops['category'])
 shops['shop_city'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id', 'shop_category', 'shop_city']] 
<feature_engineering>
train = np.loadtxt('/kaggle/input/digit-recognizer/train.csv', delimiter=',', skiprows=1)
 test = np.loadtxt('/kaggle/input/digit-recognizer/test.csv', delimiter=',', skiprows=1 )
cats['type_code'] =(
 cats['item_category_name'] 
.apply(
 lambda x: x.split(' ')[0] 
)
.astype(str)
)
 cats.loc[ 
(cats['type_code'] == 'Игровые')| 
(cats['type_code'] == 'Аксессуары'),
'category'
] = 'Игры' 
<feature_engineering>
train_label = train[:, 0]
train_img = np.resize(train[:, 1:],(train.shape[0], 28, 28, 1))
test_img = np.resize(test,(test.shape[0], 28, 28, 1))
categories = []
for categ in cats['type_code'].unique() :
if len(cats[cats['type_code'] == categ])> 4:
categories.append(categ)
 cats['type_code'] = cats['type_code'].apply(lambda x: x if x in categories else 'etc' )<categorify>
x_train, x_val, y_train, y_val = train_test_split(train_img, train_label, test_size=0.2, random_state=42 )
cats['type_code'] = LabelEncoder().fit_transform(cats['type_code'])
 cats['split'] =(
 cats['item_category_name'] 
.apply(lambda x: x.split('-')) 
)
 cats['subtype'] =(
 cats['split'] 
.apply(
 lambda x: x[1].strip() if len(x)>= 2 else x[0].strip() 
)
)
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id', 'subtype_code', 'type_code']]<categorify>
epochs = 50
batch_size = 128
validation_steps = 10000
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('\W+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(filters=64,
kernel_size=(3,3),
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2),
input_shape=x_train.shape[1:]))
model.add(tf.keras.layers.Conv2D(filters=64,
kernel_size=3,
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.Conv2D(filters=64,
kernel_size=(3,3),
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.Conv2D(filters=128,
kernel_size=(3,3),
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Conv2D(filters=128,
kernel_size=(3,3),
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.Conv2D(filters=192,
kernel_size=(3,3),
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Conv2D(filters=192,
kernel_size=(5,5),
padding='same',
activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(256, activation=partial(tf.nn.leaky_relu, alpha=1e-2)))
 model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
items['name1'], items['name2'] = items['item_name'].str.split('[', 1 ).str
items['name1'], items['name3'] = items['item_name'].str.split('(', 1 ).str
items['name2'] = items['name2'].str.replace('\W+', ' ' ).str.lower()
items['name3'] = items['name3'].str.replace('\W+', ' ' ).str.lower()
items = items.fillna('0')
 items['item_name'] = items['item_name'].apply(lambda x: name_correction(x))
items['name2'] = items['name2'].apply(lambda x: x[:-1] if x != '0' else '0' )<feature_engineering>
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4, epsilon=1e-8),
loss='categorical_crossentropy',
metrics=['accuracy'] )
items['type'] =(
 items['name2'] 
.apply(
 lambda x: x[0:8] if x.split(' ')[0] == 'xbox' else x.split(' ')[0] 
)
)
 items.loc[ 
(items['type'] == 'x360')| 
(items['type'] == 'xbox360')| 
(items['type'] == 'xbox 360'),
'type'
] = 'xbox 360'
items.loc[items['type'] == '', 'type'] = 'mac'
items.type =(
 items['type'] 
.apply(
 lambda x: x.replace(' ', '')
)
)
 items.loc[ 
(items['type'] == 'pc')| 
(items['type'] == 'pс')| 
(items['type'] == 'pс'),
'type'
] = 'pс'
items.loc[items['type'] == 'рs3' , 'type'] = 'рs3'<drop_column>
data_aug = tf.keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False )
group_sum =(
 items 
.groupby('type')
.agg({'item_id': 'count'})
.reset_index() 
)
 drop_cols = []
for categ in group_sum['type'].unique() :
if group_sum.loc[(group_sum['type'] == categ), 'item_id'].values[0] <= 39:
drop_cols.append(categ)
 items['name2'] =(
 items['name2'] 
.apply(
 lambda x: 'other' if x in drop_cols else x 
)
)
 items = items.drop(['type'], axis=1 )<categorify>
callback = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
patience=1,
verbose=2,
factor=0.5,
min_lr=1e-7 )
items['name2'] = LabelEncoder().fit_transform(items['name2'])
 items['name3'] = LabelEncoder().fit_transform(items['name3'])
 items.drop(['item_name', 'name1'], axis=1, inplace=True)
<data_type_conversions>
y_train_labels = tf.keras.utils.to_categorical(y_train)
 y_val_labels = tf.keras.utils.to_categorical(y_val )
matrix = []
cols = ['date_block_num', 'shop_id', 'item_id']
for i in range(34):
sales = train[train['date_block_num'] == i]
matrix.append(
 np.array(
 list(product(
 [i],
sales['shop_id'].unique() ,
sales['item_id'].unique() 
)) ,
dtype = np.int16 
)
)
 matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix = matrix.astype({
'date_block_num': np.int8,
'shop_id': np.int8,
'item_id': np.int16
})
 matrix.sort_values(cols, inplace=True )<feature_engineering>
train_aug = data_aug.flow(x_train, y_train_labels, batch_size=batch_size)
 val_aug = data_aug.flow(x_val, y_val_labels, batch_size=batch_size )
train['revenue'] = train['item_cnt_day'] * train['item_price']<merge>
hist = model.fit(train_aug,
steps_per_epoch=x_train.shape[0]//batch_size,
epochs=epochs,
validation_data=val_aug,
validation_steps=validation_steps//batch_size,
callbacks=[callback] )
group =(
 train 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day': 'sum'
})
)
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(
 matrix['item_cnt_month'] 
.fillna(0)
.astype(np.float16)
 )<data_type_conversions>
y_pred = model.predict(x_val)
 y_pred_labels = np.argmax(y_pred, axis=1 )
test['date_block_num'] = 34
test =(
 test 
.astype({
'date_block_num': np.int8,
'shop_id': np.int8,
'item_id': np.int16
})
 )<concatenate>
print(classification_report(y_val, y_pred_labels))
matrix = pd.concat(
 [matrix, test.drop(['ID'], axis=1)],
ignore_index=True, sort=False, keys=cols 
)
 matrix.fillna(0, inplace=True )<merge>
y_pred_test = model.predict(test_img)
 y_pred_test_labels = np.argmax(y_pred_test, axis=1 )
matrix = pd.merge(matrix, shops, on='shop_id', how='left')
 matrix = pd.merge(matrix, items, on='item_id', how='left')
 matrix = pd.merge(matrix, cats, on='item_category_id', how='left')
 matrix =(
 matrix 
.astype({
'shop_city': np.int8,
'shop_category': np.int8,
'item_category_id': np.int8,
'subtype_code': np.int8,
'name2': np.int8,
'name3': np.int16,
'type_code': np.int8
})
 )<merge>
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission['Label'] = y_pred_test_labels.astype(int)
 submission.head(10 )
def lag_feature(df, lags, cols):
for col in cols:
tmp = df[['date_block_num', 'shop_id', 'item_id', col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'shop_id', 'item_id', col + "_lag_" + str(i)]
shifted['date_block_num'] = shifted['date_block_num'] + i
df = pd.merge(df, shifted, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return df<concatenate>
submission.to_csv('submission.csv', index=False, header=True )