Kaggle competition: Digit Recognizer
del cum
gc.collect()<categorify>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 train.head()
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )<data_type_conversions>
Y_train = to_categorical(train['label'].values, 10)
 X_train =(train.loc[:, 'pixel0':] / 255 ).values
X_train.shape, Y_train.shape
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16')
 content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'])
 train_df.dtypes 
<load_from_csv>
X_test =(test / 255 ).values
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'])
 questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8')
 questions_df['tags'].fillna('188', inplace=True)
<string_transform>
datagener = ImageDataGenerator(
 rotation_range=15,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
)
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result 
<categorify>
example = X_train[6].reshape(( 1, 28, 28, 1))
label = Y_train[6]
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform)
 questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict)
 questions_df.drop(columns=['tags'], inplace=True)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 questions_df.rename(columns={'question_id':'content_id'}, inplace=True)
 questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
 del content_explation_agg
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32')
 questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16')
 del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect() 
<categorify>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16')
 part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
<drop_column>
count_network = 5
size_for_network = X_train.shape[0] // count_network
X_train_list = []
X_valid_list = []
Y_train_list = []
Y_valid_list = []
for i in range(count_network):
X_train_list.append(X_train[i * size_for_network :(i + 1)* size_for_network])
 Y_train_list.append(Y_train[i * size_for_network :(i + 1)* size_for_network])
 X_valid_list.append(
 np.concatenate(( X_train[0 * size_for_network : i * size_for_network],
X_train[(i + 1)* size_for_network : count_network * size_for_network])) 
)
 Y_valid_list.append(
 np.concatenate(( Y_train[0 * size_for_network : i * size_for_network],
Y_train[(i + 1)* size_for_network : count_network * size_for_network]))
)
del content_agg
del bundle_agg
del part_agg
gc.collect() 
<define_variables>
def build_model(lr):
model = models.Sequential()
model.add(Conv2D(96, 3, activation='relu', padding='same', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(SpatialDropout2D(0.4))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(160, 3, activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(SpatialDropout2D(0.4))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(256, 3, activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(SpatialDropout2D(0.4))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(128, 3, activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(SpatialDropout2D(0.4))
model.add(MaxPooling2D(( 2, 2)))
 model.add(Conv2D(64, 3, activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(SpatialDropout2D(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(96, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer=optimizers.Adam(lr=lr),
loss='categorical_crossentropy', metrics=['categorical_accuracy'])
 return model
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
<init_hyperparams>
list_models = [build_model(lr=1e-2)for _ in range(count_network)]
list_history = []
for i in range(count_network):
checkpoint_path = f'bestmodel{i + 1}.hdf5'
checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_categorical_accuracy',
verbose=0, save_best_only=True, mode='max')
 scheduler = LearningRateScheduler(lambda epoch, lr: lr * 0.99, verbose=0)
 early_stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=50, mode='min', verbose=0)
 tqdm_callback = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,
leave_overall_progress=True,
show_epoch_progress=False,
show_overall_progress=True)
 callbacks_list = [
checkpoint,
scheduler,
tqdm_callback,
early_stop
]
print(f'Training {i + 1} network' + '
' + '-' * 80)
 history = list_models[i].fit_generator(datagener.flow(X_train_list[i], Y_train_list[i], batch_size=50),
epochs=350, steps_per_epoch=X_train_list[i].shape[0] // 50,
callbacks=callbacks_list,
verbose=0, validation_data=(X_valid_list[i], Y_valid_list[i]))
list_history.append(history )
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))<drop_column>
for i in range(count_network):
list_models[i].load_weights(f'bestmodel{i + 1}.hdf5')
 print(f'Model №{i + 1}')
 _, acc = list_models[i].evaluate(X_train_list[i], Y_train_list[i])
 _, acc2 = list_models[i].evaluate(X_valid_list[i], Y_valid_list[i])
 print()
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
def get_predict(models, data, method_voting='soft', count_classes=10):
if method_voting == 'soft':
for_test = np.zeros(( data.shape[0], count_classes))
for i in range(len(models)) :
for_test += models[i].predict(data)
 return np.argmax(for_test, axis=1)
 elif method_voting == 'hard':
for_test = np.zeros(( data.shape[0], len(models)))
 for i in range(len(models)) :
for_test[:, i] += models[i].predict_classes(data)
 return np.int32(mode(for_test, axis=1)[0].T.flatten() )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect() 
<import_modules>
submit = pd.DataFrame(get_predict(list_models, X_test), columns=['Label'], index=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')['ImageId'])
 submit2 = pd.DataFrame(get_predict(list_models, X_test, method_voting='hard'), columns=['Label'],
index=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')['ImageId'])
 submit.index.name = 'ImageId'
submit.to_csv('submittion.csv')
 submit2.index.name = 'ImageId'
submit2.to_csv('submittion2.csv' )
<choose_model_class><EOS> 
comparison = submit.join(submit2, lsuffix='_1', rsuffix='_2')
 comparison.loc[~(comparison['Label_1'] == comparison['Label_2'])]