Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<prepare_x_and_y>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from keras import Sequential
from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D, Dropout
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
sparse_matrix_list =(train_name, train_des, train_brand_name, train_condition,
train_shipping, train_cat_1, train_cat_2, train_cat_3)
 X_train = hstack(sparse_matrix_list ).tocsr()
X_train<prepare_x_and_y>
path = '/kaggle/input/digit-recognizer/'
data = pd.read_csv(path +'train.csv')
 print(data.shape )
y_train = np.log1p(train['price'])
 y_train<train_model>
label = tf.keras.utils.to_categorical(data['label'].values)
 X = data.drop('label',axis=1 ).values
num_images = X.shape[0]
m = X.shape[1]
max_pixel = 255
img_dim = np.sqrt(m ).astype(int)
 out_dim = label.shape[1]
print('Number of different labels/output dimension ' + str(out_dim))
X = X.reshape(( num_images,img_dim,img_dim,1)) / max_pixel
print('Shape of input data X ' + str(X.shape))
Ridge_model = Ridge()
Ridge_model.fit(X_train, y_train )<concatenate>
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )
sparse_matrix_list =(test_name, test_des, test_brand_name, test_condition,
test_shipping, test_cat_1, test_cat_2, test_cat_3)
 X_test = hstack(sparse_matrix_list ).tocsr()<predict_on_test>
def create_model() :
my_model = Sequential()
my_model.add(Conv2D(32,
kernel_size =(3,3),
activation = 'relu',
input_shape =(img_dim,img_dim,1)))
 my_model.add(Dropout(0.5))
my_model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding="valid"))
my_model.add(Conv2D(32, kernel_size =(3,3), activation = 'relu'))
my_model.add(Dropout(0.5))
my_model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding="valid"))
my_model.add(Flatten())
 my_model.add(Dense(units = 500))
my_model.add(Dense(out_dim, activation = 'softmax'))
return my_model
preds = Ridge_model.predict(X_test)
 preds<prepare_output>
with tpu_strategy.scope() :
my_model = create_model()
my_model.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'] )
preds = np.expm1(preds)
 preds<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(X, label, test_size = 0.1 )
submission = pd.read_csv('sample_submission_stg2.csv')
 submission<prepare_output>
BATCH_SIZE = 128 * tpu_strategy.num_replicas_in_sync
EPOCHS = 100
STEPS_PER_EPOCH = num_images // BATCH_SIZE
submission.loc[:, 'price'] = preds
submission<save_to_csv>
history = my_model.fit(X_train, y_train ,
epochs = EPOCHS,
steps_per_epoch = STEPS_PER_EPOCH,
validation_data =(X_val, y_val))
submission.to_csv('submission.csv', index=False )<save_to_csv>
data_test = pd.read_csv(path+'test.csv' ).values
num_test = data_test.shape[0]
data_test = data_test.reshape(( num_test,img_dim,img_dim,1)) / max_pixel
submission.to_csv('submission.csv', index=False )<import_modules>
datagen = ImageDataGenerator(rotation_range=20,
zoom_range = 0.20,
width_shift_range=0.15,
height_shift_range=0.15 )
import time
from scipy.sparse import csr_matrix, hstack
from sklearn.preprocessing import LabelEncoder, Normalizer
from keras.callbacks import ModelCheckpoint
from sklearn.model_selection import train_test_split
import sys
import os
import random
import numpy as np
from keras import backend as K
from nltk.corpus import stopwords
import re
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Input, Dropout, Dense, concatenate, GRU, Embedding, Flatten, Activation
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K<load_pretrained>
Train_x, Train_y = None, None
batch = 0
for x_batch, y_batch in datagen.flow(X_train, y_train,
batch_size=BATCH_SIZE, shuffle=False):
if batch == 0:
Train_x, Train_y = x_batch, y_batch
elif batch >= len(datagen.flow(X_train, y_train,
batch_size=BATCH_SIZE)) :
break
else:
Train_x = np.concatenate(( Train_x, x_batch))
Train_y = np.concatenate(( Train_y, y_batch))
batch += 1
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z 
<define_variables>
NUM_BRANDS = 4500
NUM_CATEGORIES = 1250<compute_test_metric>
with tpu_strategy.scope() :
my_model_aug = create_model()
my_model_aug.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'] )
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)- np.log1p(y0), 2)) )<load_from_csv>
history_aug = my_model_aug.fit(Train_x, Train_y,
epochs = EPOCHS,
steps_per_epoch = STEPS_PER_EPOCH,
validation_data =(X_val, y_val))
train_df = pd.read_table("train.tsv")
 test_df = pd.read_csv(".. /input/mercari-price-suggestion-challenge/test_stg2.tsv.zip" , sep='\t')
<string_transform>
my_predictions = my_model.predict(data_test)
 imageId = np.arange(len(my_predictions))
results = my_predictions.argmax(axis=1 )
def split_cat(text):
try:
return text.split("/")
 except:
return("missing", "missing", "missing" )<categorify>
submission = pd.DataFrame(np.array([imageId + 1,results] ).transpose() , columns = ['ImageId','Label'])
 submission.to_csv('submission.csv', index = False )
def handle_missing_inplace(dataset):
dataset['general_cat'].fillna(value='missing', inplace=True)
 dataset['subcat_1'].fillna(value='missing', inplace=True)
 dataset['subcat_2'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='No description yet', inplace=True )<count_values>
my_predictions_aug = my_model_aug.predict(data_test)
 imageId = np.arange(len(my_predictions_aug))
results_aug = my_predictions_aug.argmax(axis=1 )
<data_type_conversions><EOS> 
submission_aug = pd.DataFrame(np.array([imageId + 1,results_aug] ).transpose() , columns = ['ImageId','Label'])
 submission_aug.to_csv('submission_aug.csv', index = False )