Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<install_modules>
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<import_modules>
df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df.shape
import numpy as np
import random
import pandas as pd
import joblib<set_options>
train_data = train_data.to_numpy()
train_labels = train_labels.to_numpy()
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
train_data = train_data / 255
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
filters = 64
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=4*filters, kernel_size=(4,4), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
 ])
 model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'] )
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<train_model>
history = model.fit(train_data, train_labels, epochs=40, batch_size=32, verbose=0 )
print('start handle lecture data...' )<load_from_csv>
df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 df_test.shape
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
test_data = df_test.to_numpy()
test_data = test_data / 255
test_data = test_data.reshape(( test_data.shape[0], 28, 28, 1))
test_data.shape
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
predictions = model.predict(test_data)
 predictions = np.asarray([np.argmax(prediction)for prediction in predictions])
 predictions.shape
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
df_predictions = pd.DataFrame(predictions ).rename(columns={0: "Label"})
 df_predictions.index.names = ['ImageId']
df_predictions.index += 1
df_predictions.head()
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
df_predictions.shape
df_predictions.to_csv("predictions.csv" )
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<drop_column>
df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df.shape
del(train_lectures)
 gc.collect()<categorify>
train_data = train_data.to_numpy()
train_labels = train_labels.to_numpy()
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )<data_type_conversions>
train_data = train_data / 255
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
<set_options>
filters = 64
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=4*filters, kernel_size=(4,4), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
 ])
 model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'] )
del cum
gc.collect()<train_model>
history = model.fit(train_data, train_labels, epochs=40, batch_size=32, verbose=0 )
print('start handle train_df...' )<data_type_conversions>
df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 df_test.shape
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )<groupby>
test_data = df_test.to_numpy()
test_data = test_data / 255
test_data = test_data.reshape(( test_data.shape[0], 28, 28, 1))
test_data.shape
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )<rename_columns>
predictions = model.predict(test_data)
 predictions = np.asarray([np.argmax(prediction)for prediction in predictions])
 predictions.shape
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']<data_type_conversions>
df_predictions = pd.DataFrame(predictions ).rename(columns={0: "Label"})
 df_predictions.index.names = ['ImageId']
df_predictions.index += 1
df_predictions.head()
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )<train_model>
df_predictions.shape
df_predictions.to_csv("predictions.csv" )