Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<merge>
random_seed = 2
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep' )
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
X = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 Y = X["label"]
X.drop("label",axis = 1, inplace = True)
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
np.sum(X.isnull().any() )
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
sum(test.isnull().any() )
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
def get_model(optim, loss):
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.compile(optimizer= optim, loss=loss, metrics=['accuracy'])
 return model
def normalize(X, Y = None):
X = X / 255.0
X = X.values.reshape(-1,28,28,1)
 if Y is not None:
Y = Y.values.reshape(-1,1)
 return X,Y
def get_samples(X, Y, encoder, val_size = 0.9):
X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size = val_size, random_state=42, stratify= Y)
 X_train = np.vstack([X_train,X_train,X_train])
 y_train = np.concatenate([y_train, y_train, y_train])
 idx = np.arange(0, X_train.shape[0])
 np.random.shuffle(idx)
 X_train = X_train[idx,:]
y_train = y_train[idx]
X_train = X_train.reshape(-1,28,28,1)
 X_val = X_val.reshape(-1,28,28,1)
 y_train = encoder.transform(y_train ).todense()
y_val = encoder.transform(y_val ).todense()
return X_train,X_val,y_train,y_val
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
X_train,Y_train = normalize(X,Y)
 X_test,_ = normalize(test)
 encoder = OneHotEncoder()
encoder.fit(Y_train )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
data_generator = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
EPOCHS = 50
BATCH_SIZE = 20
ENSEMBLES = 7
results = np.zeros(( test.shape[0],10))
histories = []
models = []
callback_list = [
ReduceLROnPlateau(monitor='val_loss', factor=0.25, min_lr=0.00001, patience=2, verbose=1),
EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=3, verbose=1)
 ]
optimizer = Adam(learning_rate=0.001)
 loss_fn = CategoricalCrossentropy()
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
for i in range(ENSEMBLES):
X_train_tmp, X_val, y_train_tmp, y_val = get_samples(X_train, Y_train, encoder)
 data_generator.fit(X_train_tmp)
 models.append(get_model(optimizer, loss_fn))
history = models[i].fit_generator(data_generator.flow(X_train_tmp, y_train_tmp, batch_size=BATCH_SIZE),
epochs=EPOCHS,
callbacks=[callback_list],
validation_data=(X_val, y_val),
steps_per_epoch=X_train_tmp.shape[0] // BATCH_SIZE,
use_multiprocessing=True,
verbose = 2)
 histories.append(history )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
results = np.zeros(( X_test.shape[0],10))
for i in range(ENSEMBLES):
results = results + models[i].predict(X_test)
 results = np.argmax(results, axis = 1 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
submission = pd.DataFrame({"ImageID": range(1,len(results)+1), "Label": results})
 submission.to_csv('submission.csv', index=False )