Kaggle competition: Digit Recognizer
items = items.drop(['item_name'], axis=1 )<groupby>
sns.set()
items['first_sale_date'] = sales_train.groupby('item_id' ).agg({'date_block_num': 'min'})['date_block_num']
items.head()<count_missing_values>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 train
items[items['first_sale_date'].isna() ]<data_type_conversions>
X_train = train.drop(['label'],axis = 1)
 y_train = train['label']
items['first_sale_date'] = items['first_sale_date'].fillna(34 )<feature_engineering>
X_test = test
item_categories['category'] = item_categories['item_category_name'].apply(lambda x: x.split() [0] )<count_values>
print('Number of null values in training set is : ',train.isnull().sum().unique())
 print('Number of null values in test set is : ',test.isnull().sum().unique() )
item_categories['category'].value_counts()<feature_engineering>
X_train_reshaped = X_train.values.reshape(-1,28,28,1)
 X_test_reshaped = X_test.values.reshape(-1,28,28,1)
 X_train_normalised = X_train_reshaped/255.
 X_test_normalised = X_test_reshaped/255 .
def make_etc(x):
if len(item_categories[item_categories['category']==x])>= 5:
return x
else:
return 'etc'
item_categories['category'] = item_categories['category'].apply(make_etc )<categorify>
y_train_encoded = to_categorical(y_train,num_classes=10 )
label_encoder = LabelEncoder()
item_categories['category'] = label_encoder.fit_transform(item_categories['category'])
 item_categories = item_categories.drop('item_category_name', axis=1 )<merge>
X_train_final,X_val,y_train_final,y_val = train_test_split(X_train_normalised,y_train_encoded,test_size = 0.2,random_state = 42 )
group = sales_train.groupby(idx_features ).agg({'item_cnt_day': 'sum',
'item_price': 'mean'})
 group = group.reset_index()
group = group.rename(columns={'item_cnt_day': 'item_cnt_month', 'item_price': 'item_price_mean'})
 train = train.merge(group, on=idx_features, how='left')
 train.head()<set_options>
model = Sequential()
model.add(Conv2D(input_shape=(28,28,1),filters=64,kernel_size=(3,3),padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(Conv2D(filters=32,kernel_size=(3,3),padding="same", activation="relu"))
model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=64, kernel_size=(3,3), padding="same", activation="relu"))
model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"))
model.add(Dropout(0.3))
model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
 model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(units=256,activation="relu"))
model.add(Dropout(0.3))
model.add(Dense(units=10, activation="softmax"))
del group
gc.collect() ;<merge>
model.compile(loss = 'categorical_crossentropy',optimizer = 'adam',metrics = ['accuracy'])
 model.summary()
group = sales_train.groupby(idx_features ).agg({'item_cnt_day': 'count'})
 group = group.reset_index()
group = group.rename(columns={'item_cnt_day': 'item_count'})
 train = train.merge(group, on=idx_features, how='left')
 del group, sales_train
gc.collect()
train.head()<categorify>
callbacks = [
EarlyStopping(monitor = 'loss', patience = 6),
ReduceLROnPlateau(monitor = 'loss', patience = 4)
 ]
test['date_block_num'] = 34
all_data = pd.concat([train, test.drop('ID', axis=1)],
ignore_index=True,
keys=idx_features)
 all_data = all_data.fillna(0)
 all_data.head()<merge>
model.fit(X_train_final,y_train_final,
batch_size = 64,
epochs = 100,
verbose = 1,
validation_data =(X_val,y_val),
callbacks = callbacks )
all_data = all_data.merge(shops, on='shop_id', how='left')
 all_data = all_data.merge(items, on='item_id', how='left')
 all_data = all_data.merge(item_categories, on='item_category_id', how='left')
 all_data = downcast(all_data)
 del shops, items, item_categories
gc.collect() ;<prepare_output>
score = model.evaluate(X_val,y_val,verbose = 0)
 print('The loss on validation set is {0} and the accuracy is {1}'.format(round(score[0],3),round(score[1],3)) )
def resumetable(df):
print(f'Data Shape: {df.shape}')
 summary = pd.DataFrame(df.dtypes, columns=['Dtypes'])
 summary['Null'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First_values'] = df.loc[0].values
summary['Second_values'] = df.loc[1].values
summary['Third_values'] = df.loc[2].values
return summary<groupby>
results = model.predict(X_test_normalised)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name='Label')
def add_mean_features(df, mean_features, idx_features):
assert(idx_features[0] == 'date_block_num')and \
len(idx_features)in [2, 3]
if len(idx_features)== 2:
feature_name = idx_features[1] + '_mean_sales'
else:
feature_name = idx_features[1] + '_' + idx_features[2] + '_mean_sales'
group = df.groupby(idx_features ).agg({'item_cnt_month': 'mean'})
 group = group.reset_index()
group = group.rename(columns={'item_cnt_month': feature_name})
 df = df.merge(group, on=idx_features, how='left')
 df = downcast(df, False)
 mean_features.append(feature_name)
 del group
gc.collect()
return df, mean_features<feature_engineering>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_result.csv",index=False )