Kaggle competition: Digit Recognizer
train_df['lag'] = train_df.groupby(['user_id','part'])[target].shift()
cum = train_df.groupby(['user_id','part'])['lag'].agg(['cumsum','cumcount'])
 cum.fillna(0,inplace=True)
 train_df.drop(columns="lag",inplace=True)
 train_df['mean_part_user'] =(cum['cumsum']/cum['cumcount'] ).astype("float16")
 del cum 
<groupby>
submission = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
 submission.head()
user_agg_part = train_df.groupby(['user_id','part'])[target].agg(['sum','count'] )<drop_column>
traindat = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 traindat.head()
train_df.drop(columns="part",inplace=True )<load_from_csv>
testdat = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 testdat.head()
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
Y = traindat['label']
X = traindat.drop(['label'], axis = 1)
 X.head()
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
Y = np.array(Y)
 X = np.array(X)/255.
 X = np.array([x.reshape(( 28, 28, 1)) for x in X])
 X = np.append(X, X, axis = 0)
 Y = np.append(Y, Y, axis = 0)
 X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.2, random_state = 40)
 print(X_train.shape)
 print(X_test.shape)
 print(y_train.shape)
 print(y_test.shape )
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
model = Sequential([
Conv2D(input_shape =(28,28, 1), filters = 128, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
MaxPooling2D(pool_size =(2,2)) ,
Conv2D(filters = 128, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
Conv2D(filters = 128, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
MaxPooling2D(pool_size =(2,2)) ,
Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'),
BatchNormalization() ,
Flatten() ,
Dense(128, activation = 'relu'),
BatchNormalization() ,
Dropout(0.5),
Dense(64, activation = 'relu'),
Dense(10, activation = 'softmax')
 ])
 model.compile(optimizer = 'adam', metrics = ['accuracy'], loss = 'sparse_categorical_crossentropy')
 model.summary()
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
history = model.fit(X_train, y_train,batch_size = 10000, epochs = 35, validation_data =(X_test, y_test))
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
model.fit(X, Y, epochs = 10 )
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<filter>
test_preds = model.predict(testdat)
 print(test_preds.shape)
 test_preds = np.array([np.argmax(y)for y in test_preds])
 print(test_preds.shape)
train_lectures[train_lectures.user_id==5382]<filter>
testId = [*range(1, len(test_preds)+1)]
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]<drop_column>
output = pd.DataFrame({'ImageId': testId,
'Label': test_preds})
 output.to_csv('submission.csv', index=False)