Kaggle competition: Digit Recognizer
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
%matplotlib inline
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
X = train.drop('label', axis = 1)
 y = train['label']
X = X.values.reshape(-1, 28, 28, 1)
 X = X / 255
y = to_categorical(y)
 plt.imshow(X[1][:,:,0])
 print(str(y[1]))
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<data_type_conversions>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, random_state = 123 )
train_df = train_df.astype(data_types_dict )<groupby>
datagen = ImageDataGenerator(zoom_range = 0.1, width_shift_range = 0.1, height_shift_range = 0.1, rotation_range = 10 )
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'] )<data_type_conversions>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides =(2,2)))
 model.add(Dropout(0.4))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(5, 5), activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides =(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(512, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(1024, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(10, activation = 'softmax'))
train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] ).astype("float16" )<data_type_conversions>
model.compile(optimizer = 'adam', metrics = ['accuracy'], loss = 'categorical_crossentropy' )
train_df["lag"]=0
train_df.loc[train_df[target] ==0, 'lag'] = train_df['content_correctness']
train_df['lag'] = train_df.groupby(['user_id'])["lag"].shift()
cum = train_df.groupby(['user_id'])['lag'].agg(['cumsum','cumcount'])
 cum.fillna(0,inplace=True)
 train_df["count_user"]=cum["cumcount"].astype("int16")
 train_df['mean_fail_user'] =(cum['cumsum']/(train_df["count_user"])).astype("float16")
 del cum
train_df["lag_std"]=0
train_df['lag'] = train_df.groupby(['user_id'])["lag"].shift(-1)
 train_df.loc[train_df[target] ==0, 'lag_std'] =(( train_df["mean_fail_user"]-train_df["lag"])**2 ).astype("float16")
 train_df["lag_std"] = train_df.groupby(['user_id'])["lag_std"].shift()
cum_std = train_df.groupby(['user_id'])['lag_std'].agg(['cumsum'])
 train_df.drop(columns=['lag_std'], inplace=True)
 train_df.drop(columns=['lag'], inplace=True)
 train_df['deviation_fail_user'] =(cum_std['cumsum']/(train_df["count_user"])).astype("float16")
 del cum_std 
<categorify>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=3, verbose=1, factor=0.5, min_lr=0.00001 )
train_df['dev_fail_not_divided']=0
train_df.loc[train_df[target] ==0, 'dev_fail_not_divided'] =(( train_df.loc[train_df[target] ==0, 'mean_fail_user']- train_df.loc[train_df[target] ==0, 'content_correctness'])**2 ).astype("float16")
 user_agg_std_fail = train_df.loc[train_df[target]==0].groupby(["user_id"])[ 'dev_fail_not_divided'].agg(['sum'])
 train_df.drop(columns=['dev_fail_not_divided'], inplace=True)
 user_agg_fail = train_df.loc[train_df[target]==0].groupby(["user_id"])['content_correctness'].agg(['sum'] )<load_from_csv>
hist = model.fit_generator(datagen.flow(X_train, y_train, batch_size = 32), epochs = 35, validation_data =(X_test, y_test), callbacks = [learning_rate_reduction] )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1)
 questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<merge>
final_loss, final_acc = model.evaluate(X_test, y_test, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
train_df=pd.merge(train_df,questions_df[["content_id","part"]],on="content_id",how="left" )<drop_column>
test = test.values.reshape(-1, 28, 28, 1)/ 255
y_pred = model.predict(test, batch_size = 64)
 y_pred = np.argmax(y_pred,axis = 1)
 y_pred = pd.Series(y_pred,name="Label")
 y_pred
train_df.drop(columns='content_correctness',inplace=True )<data_type_conversions>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),y_pred],axis = 1)
 submission.to_csv("submission.csv",index=False )