Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<data_type_conversions>
%matplotlib inline
np.random.seed(0 )
train.prior_question_elapsed_time =(train.prior_question_elapsed_time.fillna(0)).astype(int)
 train.prior_question_had_explanation =(train.prior_question_had_explanation.fillna(False)* 1 ).astype(int)
 train.content_type_id =(train.content_type_id * 1 ).astype(int)
 train = reduce_mem_usage(train )<drop_column>
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools
train_lec = train.query("content_type_id==1" ).copy()
train_lec.reset_index(drop=True, inplace=True)
 train.drop(train.loc[train.content_type_id==1].index, inplace=True)
 train.reset_index(drop=True, inplace=True)
 train.rename(columns={"content_id": "question_id"}, inplace=True)
 train_lec.rename(columns={"content_id": "question_id"}, inplace=True)
 gc.collect()
train.shape, train_lec.shape<categorify>
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
def cut(df, t_col, n_col):
df = df.copy()
df[n_col] = pd.cut(df[t_col], [-0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1], labels=False)
 df[n_col] = df[n_col].fillna(99 ).astype("int8")
 return df
def add_que_rate_class(df):
df = df.copy()
df["question_rate_class"] = 0
for sta, end in zip(np.arange(0.3, 1.0, 0.1), np.arange(0.4, 1.1, 0.1)) :
sta = round(sta,1)
 end = round(end,1)
 end = 1.1 if end==1 else end
df.loc[(sta <= df.question_rate)&(df.question_rate < end), "question_rate_class"] = int(sta*10)
 return df
def get_user_items_dict(ary, sizes_cumsum):
end = sizes_cumsum
sta = np.hstack([np.array([0]), end[:-1]])
 users = np.unique(ary[:,0])
 d = {}
for u, s, e in zip(users, sta, end):
d[u] = ary[s:e, 1:]
return d
def change_rate_with_num(df, t_col, n_col, method="respectively", n1=5, n2=3):
df = df.copy()
if method == "respectively":
df.loc[(df[n_col] < n1)&(df[t_col] <= 0.3), t_col] = 0.3
df.loc[(df[n_col] < n1)&(0.3 < df[t_col])&(df[t_col] <= 0.7), t_col] = 0.5
df.loc[(df[n_col] < n1)&(0.7 < df[t_col]), t_col] = 0.7
else:
df.loc[(df[n_col] < n2), t_col] = 0.5
return df
def change_rate_high_low(df, t_col, h=0.9, l=0.1):
df = df.copy()
df.loc[df[t_col] < l, t_col] = l
df.loc[df[t_col] > h, t_col] = h
return df<count_unique_values>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 X = train_data.drop(labels = ['label'], axis= 1)
 y = train_data['label']
del train_data
sns.countplot(y )
CV = 4
MODEL_NUM = 1
USE_USER_NUM = 10000
if DEBUG:
USE_USER_NUM = 500
if TUNE:
TRAIN_RATE, TUNE_RATE, TEST_RATE = 0.4, 0.3, 0.3
else:
TRAIN_RATE, TEST_RATE = 0.7, 0.3
print("%s users reduce to %s" %(train.user_id.nunique() , USE_USER_NUM))
user_ids = train.user_id.unique()
random.shuffle(user_ids)
 ext_user = user_ids[:USE_USER_NUM]
ext_user_bol = np.isin(np.array(train.user_id), np.sort(ext_user))
sizes_quser = np.array(train.user_id.value_counts().sort_index().values, dtype="int16")
 sizes_qcumsum = sizes_quser.cumsum()
unique_user_ids = np.array(train.user_id)[sizes_qcumsum-1]<feature_engineering>
X = X/255.0
test_data = test_data/255.0
df_etime = pd.DataFrame({"question_id" : np.array(train.question_id[:-1]),
"prior_question_elapsed_time": np.array(train.prior_question_elapsed_time[1:])})
 df_etime = df_etime.loc[df_etime.prior_question_elapsed_time!=0].copy()
df_etime.prior_question_elapsed_time = df_etime.prior_question_elapsed_time /(1000*3600)
 df_etime["etime_kurt"] = df_etime.prior_question_elapsed_time
df_etime["etime_std"] = df_etime.prior_question_elapsed_time
df_etime["etime_skew"] = df_etime.prior_question_elapsed_time
df_etime_kurt = df_etime.groupby("question_id", as_index=False ).etime_kurt.apply(pd.DataFrame.kurt)
 df_etime_rate = df_etime.groupby("question_id", as_index=False ).agg({"prior_question_elapsed_time": "mean",
"etime_std" : "std",
"etime_skew": "skew"})
 df_etime_rate.rename(columns={"prior_question_elapsed_time": "etime_mean"}, inplace=True)
 df_etime_rate = df_etime_rate.merge(df_etime_kurt, on="question_id")
 df_etime_rate.fillna(0, inplace=True)
 df_etime_rate.question_id = df_etime_rate.question_id.astype("int32")
 df_etime_rate.etime_mean = df_etime_rate.etime_mean.astype("float32")
 df_etime_rate.etime_std = df_etime_rate.etime_std.astype("float32")
 df_etime_rate.etime_kurt = df_etime_rate.etime_kurt.astype("float32")
 df_etime_rate.etime_skew = df_etime_rate.etime_skew.astype("float32")
 d_etime = dict(zip(df_etime_rate.question_id, df_etime_rate.etime_mean))
del df_etime, df_etime_kurt
gc.collect()
print(df_etime_rate.shape)
 df_etime_rate.head(2 )<merge>
y = to_categorical(y, num_classes=10 )
df_etmie_rate = train[["row_id","user_id","question_id","prior_question_elapsed_time"]].copy()
df_etmie_rate.rename(columns={"prior_question_elapsed_time":"lag_etime"}, inplace=True)
 df_etmie_rate.lag_etime = df_etmie_rate.lag_etime /(1000*3600)
 df_etmie_rate.lag_etime = df_etmie_rate.groupby("user_id" ).lag_etime.shift(-1)
 df_etmie_rate = df_etmie_rate.merge(df_etime_rate[["question_id","etime_mean"]], on="question_id")
 df_etmie_rate.lag_etime = df_etmie_rate.lag_etime - df_etmie_rate.etime_mean
df_etmie_rate.sort_values("row_id", inplace=True)
 df_etmie_rate.reset_index(drop=True, inplace=True)
 df_etmie_rate["num"] = 1
df_etmie_rate["num"] = df_etmie_rate.groupby("user_id" ).num.shift(1)
 df_etmie_rate["num"] = df_etmie_rate.groupby("user_id" ).num.cumsum()
df_etmie_rate["num_etime"] = df_etmie_rate.groupby("user_id" ).lag_etime.shift(1)
 df_etmie_rate["num_etime"] = df_etmie_rate.groupby("user_id" ).num_etime.cumsum()
df_etmie_rate["etime_rate"] = df_etmie_rate.num_etime / df_etmie_rate.num
df_etmie_rate.fillna(0, inplace=True)
 df_etmie_rate.num_etime = df_etmie_rate.num_etime.astype("float32")
 df_etmie_rate.etime_rate = df_etmie_rate.etime_rate.astype("float32")
 df_etmie_rate.drop(["row_id","user_id","question_id","lag_etime","etime_mean","num"], axis=1, inplace=True)
 print(df_etmie_rate.shape)
 display(df_etmie_rate.head(3))
ar_etmie_rate = np.array(df_etmie_rate)
 hist_ar_etmie_rate, ar_etmie_rate = get_hist_and_training_data(ar_etmie_rate)
 hist_ar_etmie_rate, ar_etmie_rate = hist_ar_etmie_rate[:,0], ar_etmie_rate[:,1]
del df_etmie_rate
gc.collect()<categorify>
random_seed =5
X_train,X_val,y_train, y_val =train_test_split(X,y,random_state=random_seed, test_size=0.08 )
df_lecs = df_lecs.join(pd.DataFrame(pd.get_dummies(df_lecs.type_of)))
 df_lecs.rename(columns={"lecture_id" :"question_id",
"solving question":"solving_question"}, inplace=True)
 df_lecs.drop(["tag","type_of"], axis=1, inplace=True)
 df_lecs = reduce_mem_usage(df_lecs)
 tags = sum(df_qs.tags.apply(lambda x: [] if x is np.nan else x.split(" ")) , [])
 df_tags = pd.DataFrame(pd.Series(tags ).value_counts() , columns=["tag"])
 tags_num = df_tags.to_dict() ["tag"]
df_tags.reset_index(inplace=True)
 df_tags.columns = ["tag","num"]
df_tags["total"] = df_qs.shape[0]
df_tags["tfidf"] = 1 * np.log(df_tags.total / df_tags.num)
 df_tags.index = df_tags.tag
tags_tfidf = df_tags[["tfidf"]].to_dict() ["tfidf"]
df_tags = pd.DataFrame(list(df_qs.tags.apply(lambda x: [] if x is np.nan else x.split(" "))),
columns=["tag"+str(t)for t in range(1,7)])
 df_tags = df_tags.fillna(0 ).astype("int16")
 df_tags = df_qs[["question_id"]].join(df_tags)
 df_qs["tag_rel"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else sum([tags_num[t] for t in x.split(" ")]))
df_qs["tag_tfidf_sum"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else sum([tags_tfidf[t] for t in x.split(" ")]))
df_qs["num_tag"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else len(x.split(" ")))
 df_qs.drop(["correct_answer","tags"], axis=1, inplace=True)
 df_qs = df_qs.merge(df_tags, on="question_id")
 df_qs = reduce_mem_usage(df_qs)
 del df_tags, tags_num, tags_tfidf
gc.collect()
print(df_lecs.shape, df_qs.shape)
 display(df_lecs.head(2))
display(df_qs.head(2))<merge>
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
train = train.merge(df_qs[["question_id","bundle_id","part","tag1"]], on='question_id', how='left')
 train = reduce_mem_usage(train )<feature_engineering>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
df = train[["user_id","question_id","bundle_id","user_answer","answered_correctly"]].copy()
df_question_rate = df_qs[["question_id","bundle_id"]].copy()
for c in ["question_id", "bundle_id"]:
c_name = c[:-3]
df[c_name+"_kurt"] = df.user_answer
df[c_name+"_skew"] = df.user_answer
df_kurt = df.groupby(c, as_index=False)[c_name+"_kurt"].apply(pd.DataFrame.kurt)
 df_rate = df.groupby(c, as_index=False ).agg({"answered_correctly": "mean",
"user_answer" : "std",
c_name+"_skew" : "skew"})
 df_rate.rename(columns={"answered_correctly":c_name+"_rate",
"user_answer" :c_name+"_std"}, inplace=True)
 df_rate = df_rate.merge(df_kurt, on=c)
 df_question_rate = df_question_rate.merge(df_rate, on=c, how="left")
 df_question_rate.fillna(0, inplace=True)
 min2_val = sorted(df_question_rate.question_rate.unique())[1]
ins_row = df_question_rate[df_question_rate.question_rate == min2_val].iloc[:,2:]
df_question_rate.loc[df_question_rate.question_rate == 0, ins_row.columns] = ins_row.values
df_question_rate["deviation_p"] = df_question_rate[["question_rate","question_std"]].apply(lambda x: get_deviation_value(x[0],x[1], "p"), axis=1)
 df_question_rate["deviation_m"] = df_question_rate[["question_rate","question_std"]].apply(lambda x: get_deviation_value(x[0],x[1], "m"), axis=1)
 df_question_rate = add_que_rate_class(df_question_rate)
 df_question_rate.bundle_id = df_question_rate.bundle_id.astype("int16")
 df_question_rate.question_rate_class = df_question_rate.question_rate_class.astype("int8")
 df_question_rate = reduce_mem_usage(df_question_rate)
 df_part_rate = train.groupby("part", as_index=False ).answered_correctly.mean()
df_part_rate.rename(columns={"answered_correctly": "part_rate"}, inplace=True)
 df_part_rate = reduce_mem_usage(df_part_rate)
 del df, df_kurt, df_rate, min2_val, ins_row
gc.collect()
print(df_question_rate.shape, df_part_rate.shape)
 display(df_question_rate.head(2))
display(df_part_rate.head(2))<merge>
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )
df_question_rate = df_question_rate.merge(df_que_exp_rate, on="question_id", suffixes=("", "_exp"))
del df_que_exp_rate
gc.collect()<merge>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
train = train.merge(df_question_rate[["question_id","prior_question_had_explanation","question_rate_exp","deviation_p","deviation_m"]],
on=["question_id","prior_question_had_explanation"])
 train.sort_values(by="row_id", inplace=True)
 train.reset_index(drop=True, inplace=True )<data_type_conversions>
epochs = 35
batch_size = 64
ans = np.array(train.answered_correctly, dtype="int16")
 num = np.ones(train.shape[0], dtype="int16")
 exp = np.array(train.prior_question_had_explanation, dtype="int16")
 devp = ans * np.array(train.deviation_p, dtype="float64")
 devm = devp + np.where(ans==0, -1, 0)* np.array(train.deviation_m, dtype="float64")
 ans_cumsum = intervaled_cumsum(ans, sizes_qcumsum ).astype("int16")
 num_cumsum = intervaled_cumsum(num, sizes_qcumsum ).astype("int16")
 exp_cumsum = intervaled_cumsum(exp, sizes_qcumsum ).astype("int16")
 devp_cumsum = intervaled_cumsum(devp, sizes_qcumsum ).astype("float32")
 devm_cumsum = intervaled_cumsum(devm, sizes_qcumsum ).astype("float32")
 ans_rate =(ans_cumsum / num_cumsum ).astype("float32")
 study_rate =(exp_cumsum / shift_to_prior(num_cumsum, sizes_qcumsum)).astype("float32")
 study_rate[np.isnan(study_rate)] = 0
study_rate[np.isinf(study_rate)] = 0
hist_ans_cumsum, ans_cumsum = get_hist_and_training_data(ans_cumsum)
 hist_num_cumsum, num_cumsum = get_hist_and_training_data(num_cumsum)
 hist_exp_cumsum, exp_cumsum = get_hist_and_training_data(exp_cumsum)
 hist_devp_cumsum, devp_cumsum = get_hist_and_training_data(devp_cumsum)
 hist_devm_cumsum, devm_cumsum = get_hist_and_training_data(devm_cumsum)
 hist_ans_rate, ans_rate = get_hist_and_training_data(ans_rate)
 hist_study_rate, _ = get_hist_and_training_data(study_rate)
 train.drop(["deviation_p","deviation_m"], axis=1, inplace=True)
 del ans, num, exp, devp, devm
gc.collect()<categorify>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1, 
)
 datagen.fit(X_train )
%%time
num_part_cols = ["num_part"+str(p)for p in range(1,8)]
ans_part_cols = ["ans_part"+str(p)for p in range(1,8)]
part_rate_cols = ["part"+str(p)+"_rate" for p in range(1,8)]
part_cols = ans_part_cols + num_part_cols
ar_part_num = np.array(pd.get_dummies(train.part))
ar_part_ans = ar_part_num * np.array(train.answered_correctly ).reshape(-1,1)
 ar_part = np.hstack([ar_part_ans, ar_part_num] ).astype("int16")
 del ar_part_num, ar_part_ans
gc.collect()
for i in range(ar_part.shape[1]):
ar_part[:,i] = intervaled_cumsum(ar_part[:,i], sizes_qcumsum)
 ar_part_rate =(ar_part[:,:7] / ar_part[:,7:] ).astype("float32")
 np.nan_to_num(ar_part_rate, copy=False)
 hist_ar_part_rate, _ = get_hist_and_training_data(ar_part_rate)
 hist_ar_part, ar_part = get_hist_and_training_data(ar_part )<define_variables>
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
abi_cols = []
abi_rate_cols = []
for sta, end in zip([0.0] + list(np.arange(0.3, 1.0, 0.1)) ,
[0.3] + list(np.arange(0.4, 1.1, 0.1))):
sta = round(sta,1)
 end = round(end,1)
 abi_cols += ["ans_abi" +str(int(sta*10)) , "num_abi"+str(int(sta*10)) ]
abi_rate_cols += ["abi_rate"+str(int(sta*10)) ]
end = 1.1 if end == 1 else end
print("Target question rate is from %s to %s" %(sta, end))
df_ability = train[["answered_correctly","question_rate_exp"]].copy()
df_ability["num"] = 1
df_ability.loc[(df_ability["question_rate_exp"] <= sta)|(end < df_ability["question_rate_exp"]), ["answered_correctly","num"]] = 0
abians = np.array(df_ability.answered_correctly, dtype="int16")
 abinum = np.array(df_ability.num, dtype="int16")
 del df_ability
gc.collect()
abians_cumsum = intervaled_cumsum(abians, sizes_qcumsum ).astype("int16")
 abinum_cumsum = intervaled_cumsum(abinum, sizes_qcumsum ).astype("int16")
 del abians, abinum
gc.collect()
abians_num = np.array([abians_cumsum, abinum_cumsum] ).T
hist_abians_num, abians_num = get_hist_and_training_data(abians_num)
 if sta == 0:
hist_ar_ability = hist_abians_num
ar_ability = abians_num
else:
hist_ar_ability = np.hstack([hist_ar_ability, hist_abians_num])
 ar_ability = np.hstack([ar_ability, abians_num])
 del abians_cumsum, abinum_cumsum, abians_num, hist_abians_num
gc.collect()
abi_cols = sorted(abi_cols)
 abi_rate_cols = sorted(abi_rate_cols)
 hist_ar_ability = np.hstack([hist_ar_ability[:,::2], hist_ar_ability[:,1::2]])
 ar_ability = np.hstack([ar_ability[:,::2], ar_ability[:,1::2]] )<create_dataframe>
results = model.predict(test_data)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<merge><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("digit_cnn_keras.csv",index=False )