Kaggle competition: Digit Recognizer
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )<feature_engineering>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 x_test = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train.head()
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']<data_type_conversions>
x_train = x_train.to_numpy()
x_test = x_test.to_numpy()
y_train = y_train.to_numpy()
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )<data_type_conversions>
x_train = x_train.reshape(-1,28,28)
 x_test = x_test.reshape(-1,28,28)
 print("(Image)Train Inputs: " , x_train.shape)
 print("(Image)Test Inputs: " , x_test.shape )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
<set_options>
def sharpner(img):
img = Image.fromarray(img.astype('uint8'))
img = img.filter(ImageFilter.UnsharpMask(radius=2, percent=150))
return np.array(img)
 for i in range(x_train.shape[0]):
x_train[i] = sharpner(x_train[i])
 for i in range(x_test.shape[0]):
x_test[i] = sharpner(x_test[i])
 print(x_train.shape)
 print(x_test.shape)
 plt.imshow(x_train[3] )
del cum
gc.collect()<data_type_conversions>
def one_hottie(labels,C):

 One_hot_matrix = tf.one_hot(labels,C)
 return tf.keras.backend.eval(One_hot_matrix)
 y_train = one_hottie(y_train, 10)
 print("Y shape: " + str(y_train.shape))
<data_type_conversions>
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(64, 3, activation='relu', input_shape=(28,28,1),padding="same"),
tf.keras.layers.MaxPool2D(strides=2),
tf.keras.layers.Conv2D(128, 3, activation='relu',padding="same"),
tf.keras.layers.MaxPool2D(strides=2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256, 3, activation='relu',padding="same"),
tf.keras.layers.MaxPool2D(strides=2),
tf.keras.layers.Conv2D(256, 3, activation='relu',padding="same"),
tf.keras.layers.MaxPool2D(strides=2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(100,kernel_regularizer=tf.keras.regularizers.l2(0.01), activation='relu'),
tf.keras.layers.Dense(50,kernel_regularizer=tf.keras.regularizers.l2(0.01), activation='relu'),
tf.keras.layers.Dense(10, kernel_regularizer=tf.keras.regularizers.l2(0.01),activation='softmax')
 ])
 model.summary()
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
<data_type_conversions>
model.compile(optimizer='adam', loss=tf.keras.losses.CategoricalCrossentropy() , metrics=['accuracy'] )
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )<set_options>
datagen = tf.keras.preprocessing.image.ImageDataGenerator(
 rotation_range=45,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
fill_mode='nearest')
 datagen.fit(x_train)
 result = model.fit_generator(datagen.flow(x_train, y_train,
batch_size=64),
epochs=50,
workers=4 )
del cum
gc.collect()<categorify>
model.compile(optimizer=tf.keras.optimizers.Nadam(learning_rate=0.006),
loss=tf.keras.losses.CategoricalCrossentropy() ,
metrics=['accuracy'])
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )<data_type_conversions>
result = model.fit(x=x_train,
y=y_train,
batch_size=64,
epochs=50,
verbose=1,
shuffle=False,
initial_epoch=20,
validation_split=0.2 )
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<groupby>
model.compile(optimizer=tf.keras.optimizers.Nadam(learning_rate=0.0001),
loss=tf.keras.losses.CategoricalCrossentropy() ,
metrics=['accuracy'] )
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )<train_model>
result = model.fit(x=x_train,
y=y_train,
batch_size=64,
epochs=50,
verbose=1,
shuffle=False,
initial_epoch=20,
validation_split=0.2 )
print('start questions data...' )<load_from_csv>
model.compile(optimizer=tf.keras.optimizers.Nadam(learning_rate=0.00006),
loss=tf.keras.losses.CategoricalCrossentropy() ,
metrics=['accuracy'] )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)<groupby>
result = model.fit(x=x_train,
y=y_train,batch_size=64,
epochs=90,
verbose=1,
shuffle=False,
initial_epoch=50,
validation_split=0.2 )
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )<data_type_conversions>
check = model.evaluate(x_train,y_train )
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )<set_options>
preds = model.predict_classes(x_train)
 preds.shape
questions_df['tags'].fillna('188', inplace=True )<string_transform>
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 y_train = train.iloc[:,0]
y_train = y_train.to_numpy()
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result<categorify>
preds = model.predict_classes(x_test )
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )<data_type_conversions>
arr = [x for x in range(1,28001)]
label = pd.DataFrame(arr,columns = ["ImageId"])
 label["Label"] = pd.DataFrame(preds)
 label.head()
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )<drop_column>
label.to_csv('Y_test.csv',header=True,index = False )
questions_df.drop(columns=['tags'], inplace=True )<data_type_conversions>
model.save("MNIST_CNN_model_dataaug" )