Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<groupby>
%matplotlib inline
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )<load_from_csv>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True )<categorify>
y_train = df_train["label"]
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_avg'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )<drop_column>
X_train = df_train.drop(columns="label" )
valid_df = train_df.groupby('user_id' ).tail(6)
 last_df = train_df.groupby('user_id' ).tail(1)
 train_df.drop(valid_df.index, inplace=True )<categorify>
X_test = df_test
lencoder = LabelEncoder()
result_time_mean = train_df.prior_question_elapsed_time.mean()
train_df['prior_question_had_explanation'].fillna(False, inplace = True)
 train_df['prior_question_had_explanation_enc'] = lencoder.fit_transform(train_df['prior_question_had_explanation'])
 train_df['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)
 train_df['user_correctness'].fillna(0.7, inplace=True)
 train_df['part'].fillna(4, inplace=True)
 valid_df['prior_question_had_explanation'].fillna(False, inplace = True)
 valid_df['prior_question_had_explanation_enc'] = lencoder.fit_transform(valid_df['prior_question_had_explanation'])
 valid_df['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)
 valid_df['user_correctness'].fillna(0.7, inplace=True)
 valid_df['part'].fillna(4, inplace=True)
 last_df['prior_question_had_explanation'].fillna(False, inplace = True)
 last_df['prior_question_had_explanation_enc'] = lencoder.fit_transform(last_df['prior_question_had_explanation'])
 last_df['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)
 last_df['user_correctness'].fillna(0.7, inplace=True)
 last_df['part'].fillna(4, inplace=True)
<save_to_csv>
X_train = np.array(( df_train.iloc[:,1:].values ).astype('float32'))
last_df.to_csv('cnn_last_data.csv',index=True )<prepare_x_and_y>
X_test = np.array(df_test.values.astype('float32'))
X = train_df[['prior_question_elapsed_time','prior_question_had_explanation_enc','part','user_correctness','content_count','content_avg']]
X_val = valid_df[['prior_question_elapsed_time','prior_question_had_explanation_enc','part','user_correctness','content_count','content_avg']]
y = train_df[target]
y_val = valid_df[target]<count_missing_values>
X_train = X_train/255.0
X.isnull().any()<normalization>
X_test = X_test/255.0
scaler = StandardScaler()
X = scaler.fit_transform(X)
 X_val = scaler.transform(X_val )<choose_model_class>
y_train = np.array(df_train.iloc[:,0].values.astype('int32'))
K.clear_session()
X_train = X.reshape(X.shape[0], X.shape[1], 1)
 X_test = X_val.reshape(X_val.shape[0], X_val.shape[1], 1)
 model = Sequential()
model.add(Conv1D(64, 2, activation='relu', input_shape=X_train[0].shape))
model.add(Conv1D(64, 2, activation='relu', padding='causal'))
model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=Adam(learning_rate=0.01), loss='binary_crossentropy', metrics=[tf.keras.metrics.BinaryAccuracy() ] )<train_model>
y_train_encoded = to_categorical(y_train, num_classes=10 )
history = model.fit(X_train, y , epochs=30, verbose=2, batch_size=50000 )<save_model>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, y_train_encoded, test_size = 0.1, random_state=2 )
model.save('CNN.model' )<data_type_conversions>
model = Sequential()
model.add(Conv2D(filters = 32,
kernel_size =(5,5),
padding = 'Same',
activation ='relu',
input_shape =(28,28,1)))
 model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 32,
kernel_size =(5,5),
padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(filters = 64,
kernel_size =(3,3),
padding = 'Same',
activation ='relu'))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 64,
kernel_size =(3,3),
padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<categorify>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
joblib.dump(user_sum_dict, 'user_sum_dict.pkl')
 joblib.dump(user_count_dict, 'user_count_dict.pkl')
 joblib.dump(content_sum_dict, 'content_sum_dict.pkl')
 joblib.dump(content_count_dict, 'content_count_dict.pkl')
 joblib.dump(lencoder,'exp_encoder.pkl')
 joblib.dump(scaler,'cnn_scaler.pkl')
<feature_engineering>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
<categorify>
model.compile(optimizer = optimizer ,
loss = "categorical_crossentropy",
metrics=["accuracy"] )
print(sklearn.__version__)
 user_cache = dict()
env = riiideducation.make_env()<load_from_csv>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
questions_df = pd.read_csv("/kaggle/input/riiid-test-answer-prediction/questions.csv" ).set_index("question_id")
 qid_max = questions_df.index.max() + 1
questions_df["part"].value_counts()<categorify>
batches = datagen.flow(X_train, Y_train, batch_size=96)
 val_batches = datagen.flow(X_val, Y_val, batch_size=96 )
lectures_df = pd.read_csv("/kaggle/input/riiid-test-answer-prediction/lectures.csv")
 le = LabelEncoder()
le.fit(lectures_df["lecture_id"].values)
 def encode_cid(x):
return qid_max
lectures_df["part"] = 8
lectures_df["lecture_id"] = encode_cid(lectures_df["lecture_id"])
 lectures_df.set_index("lecture_id", inplace=True)
 lid_max = lectures_df.index.max() + 1
lectures_df["part"].value_counts()<categorify>
history = model.fit_generator(generator=batches,
epochs=1,
validation_data=val_batches,
validation_steps=val_batches.n,
verbose=2,
steps_per_epoch=batches.n,
callbacks=[learning_rate_reduction] )
lpart_dict = lectures_df["part"].to_dict()
qpart_dict = questions_df["part"].to_dict()
len(lpart_dict), len(qpart_dict )<normalization>
predictions = model.predict_classes(X_test, verbose=0 )
<categorify><EOS> 
submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("submission.csv", index=False, header=True )