Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )
fill(train)
 fill(validation)
 fill(test )<feature_engineering>
x_train = train.drop('label',axis = 1)
 y_train = train['label']
x_test = test
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)
 x_train = x_train.values.reshape(-1,28,28,1)
 x_val = x_val.values.reshape(-1,28,28,1)
 x_test = x_test.values.reshape(-1,28,28,1 )
def split_category(cate):
try:
sub_cate1, sub_cate2, sub_cate3 = cate.split("/")
 return sub_cate1, sub_cate2, sub_cate3
except:
return("Missing", "Missing", "Missing")
 def creat_category(data):
data['sub_category1'], data['sub_category2'], data['sub_category3'] = zip(*data['category_name'].apply(lambda x: split_category(x)) )<import_modules>
aug = ImageDataGenerator(rotation_range=5, width_shift_range=0.05, height_shift_range=0.05, zoom_range = 0.05)
 aug.fit(x_train )
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from scipy.sparse import csr_matrix, hstack<concatenate>
model = Sequential()
model.add(Conv2D(6, 5, padding="same",
activation="relu", input_shape=[28, 28, 1]))
model.add(MaxPool2D())
 model.add(BatchNormalization())
 model.add(Conv2D(16, 5, padding="same", activation="relu"))
model.add(MaxPool2D())
 model.add(BatchNormalization())
 model.add(Conv2D(120, 5, padding="same", activation="relu"))
model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(84, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation="softmax"))
model.compile(loss='sparse_categorical_crossentropy',
optimizer="sgd", metrics=["accuracy"])
 model.summary()
merge: pd.DataFrame = pd.concat([train, validation, test])
 submission: pd.DataFrame = test[['test_id']]
n_train = train.shape[0]
n_validation = validation.shape[0]
n_test = test.shape[0]<feature_engineering>
early_stop = EarlyStopping(
 patience=6, restore_best_weights=True,monitor = 'val_acc')
 model.fit(aug.flow(x_train, y_train,batch_size = 64),steps_per_epoch=len(x_train)/ 64, epochs=100,validation_data =(x_val,y_val),callbacks=[early_stop],use_multiprocessing = True, verbose = 0 )
NAME_MIN_DF = 10
MAX_FEATURES_ITEM_DESCRIPTION = 30000
def condition2vec(data):
vectorizer = LabelBinarizer(sparse_output=True)
 vec = vectorizer.fit_transform(data)
 return vec
def shipping2vec(data):
vectorizer = LabelBinarizer(sparse_output=True)
 vec = vectorizer.fit_transform(data)
 return vec
def brandname2vec(data):
vectorizer = LabelBinarizer(sparse_output=True)
 vec = vectorizer.fit_transform(data)
 return vec
def name2vec(data):
vectorizer = CountVectorizer(
 min_df=NAME_MIN_DF,
ngram_range=(1, 2),
stop_words='english' 
)
 vec = vectorizer.fit_transform(data)
 return vec
def description2vec(data):
vectorizer = TfidfVectorizer(
 max_features=MAX_FEATURES_ITEM_DESCRIPTION,
ngram_range=(1, 2),
stop_words='english' 
)
 vec = vectorizer.fit_transform(data)
 return vec
def category2vec(data):
vectorizer = CountVectorizer()
data_vec = []
for x in data:
category_vec = vectorizer.fit_transform(x)
 data_vec.append(category_vec)
 print('Convert Category ')
 return hstack(data_vec)
 def put_all_together(merge):
res = hstack((
name2vec(merge['name']),
condition2vec(merge['item_condition_id']),
brandname2vec(merge['brand_name']),
shipping2vec(merge['shipping']),
description2vec(merge['item_description']),
category2vec(( merge['sub_category1'], merge['sub_category2'], merge['sub_category3'])) 
))
return res<compute_test_metric>
y_pred = np.argmax(model.predict(x_test),axis = 1)
 submission = pd.DataFrame({"imageId": np.arange(1, len(y_pred)+1),"Label":y_pred})
all_data = all_data.tocsr()
train_X, validation_X, test_X = all_data[:n_train], all_data[n_train:n_train + n_validation], all_data[n_train + n_validation:]<prepare_x_and_y>
submission.to_csv("lenet_sub.csv",index=False )