Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import np_utils
from keras.datasets import mnist
import pandas as pd
train_data['item_description']=text_preprocessing(train_data['item_description'])
 validation_data['item_description']=text_preprocessing(validation_data['item_description'])
 test_data['item_description']=text_preprocessing(test_data['item_description'])
 train_data['name']=text_preprocessing(train_data['name'])
 validation_data['name']=text_preprocessing(validation_data['name'])
 test_data['name']=text_preprocessing(test_data['name'] )<string_transform>
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
print(train_data['item_description'].iloc[33],len(train_data['item_description'].iloc[33].split(' ')))
 print(train['item_description'].iloc[33],len(train['item_description'].iloc[33].split(' ')) )<drop_column>
X_train, X_test, y_train, y_test = train_test_split(train.iloc[:, 1:].values, train.iloc[:, 0].values, test_size=0.1 )
def clean_cat(cat_col):
cat_list = []
for i in tqdm(cat_col.values):
i = re.sub('[^A-Za-z0-9]+', ' ', i)
 i = i.replace(' ','')
 i = i.replace('&','_')
 cat_list.append(i.strip())
 return cat_list<concatenate>
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28, 1 )
train_data['sub_category1'] = clean_cat(train_data['sub_category1'])
 validation_data['sub_category1'] = clean_cat(validation_data['sub_category1'])
 test_data['sub_category1'] = clean_cat(test_data['sub_category1'])
 train_data['sub_category2'] = clean_cat(train_data['sub_category2'])
 validation_data['sub_category2'] = clean_cat(validation_data['sub_category2'])
 test_data['sub_category2'] = clean_cat(test_data['sub_category2'])
 train_data['sub_category3'] = clean_cat(train_data['sub_category3'])
 validation_data['sub_category3'] = clean_cat(validation_data['sub_category3'])
 test_data['sub_category3'] = clean_cat(test_data['sub_category3'] )<feature_engineering>
y_train = np_utils.to_categorical(y_train, 10)
 y_test = np_utils.to_categorical(y_test, 10)
 print('Data y origin ', y_train[0])
 print('Data y one-hot encoding ',y_train[0] )
train_data['brand_name'] = clean_cat(train_data['brand_name'])
 validation_data['brand_name'] = clean_cat(validation_data['brand_name'])
 test_data['brand_name'] = clean_cat(test_data['brand_name'] )<categorify>
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='sigmoid', input_shape=(28,28,1)))
 model.add(Conv2D(32,(3, 3), activation='sigmoid'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(Dense(128, activation='sigmoid'))
model.add(Dense(10, activation='softmax'))
countvectorizer=CountVectorizer().fit(train_data['sub_category1'])
 bow_cat1_train=countvectorizer.transform(train_data['sub_category1'])
 bow_cat1_val=countvectorizer.transform(validation_data['sub_category1'])
 bow_cat1_test=countvectorizer.transform(test_data['sub_category1'])
 print("Shape of sub_category1 features:")
 print(bow_cat1_train.shape)
 print(bow_cat1_val.shape)
 print(bow_cat1_test.shape)
 print("Some Features are:")
 print(countvectorizer.get_feature_names() )<categorify>
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'] )
countvectorizer=CountVectorizer().fit(train_data['sub_category2'])
 bow_cat2_train=countvectorizer.transform(train_data['sub_category2'])
 bow_cat2_val=countvectorizer.transform(validation_data['sub_category2'])
 bow_cat2_test=countvectorizer.transform(test_data['sub_category2'])
 print("Shape of sub_category2 features:")
 print(bow_cat2_train.shape)
 print(bow_cat2_val.shape)
 print(bow_cat2_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [50:60] )<categorify>
datagen = ImageDataGenerator(
 rotation_range= 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1 
)
 datagen.fit(X_train )
countvectorizer=CountVectorizer().fit(train_data['sub_category3'])
 bow_cat3_train=countvectorizer.transform(train_data['sub_category3'])
 bow_cat3_val=countvectorizer.transform(validation_data['sub_category3'])
 bow_cat3_test=countvectorizer.transform(test_data['sub_category3'])
("Shape of sub_category3 features:")
 print(bow_cat3_train.shape)
 print(bow_cat3_val.shape)
 print(bow_cat3_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [200:210] )<categorify>
train_generator = datagen.flow(X_train, y_train, batch_size = 32)
 validation_generator = datagen.flow(X_test, y_test, batch_size = 32 )
countvectorizer=CountVectorizer().fit(train_data['brand_name'])
 bow_brand_train=countvectorizer.transform(train_data['brand_name'])
 bow_brand_val=countvectorizer.transform(validation_data['brand_name'])
 bow_brand_test=countvectorizer.transform(test_data['brand_name'])
("Shape of brand_name features:")
 print(bow_brand_train.shape)
 print(bow_brand_val.shape)
 print(bow_brand_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [35:45] )<categorify>
%%time
H = model.fit_generator(train_generator,
steps_per_epoch = X_train.shape[0] // 32,
epochs=50,
validation_data = validation_generator,
validation_steps = X_test.shape[0] // 32,
verbose=1 )
countvectorizer=CountVectorizer(min_df=10 ).fit(train_data['name'])
 bow_name_train=countvectorizer.transform(train_data['name'])
 bow_name_val=countvectorizer.transform(validation_data['name'])
 bow_name_test=countvectorizer.transform(test_data['name'])
 print("After Vectorization of name feature: ")
 print(bow_name_train.shape)
 print(bow_name_val.shape)
 print(bow_name_test.shape)
 print("Some Features are: ")
 print(countvectorizer.get_feature_names() [210:220] )<categorify>
score = model.evaluate(X_test, y_test, verbose=0)
 print(score )
tfidfvectorizer=TfidfVectorizer(ngram_range=(1,2),min_df=10,max_features=5000 ).fit(train_data['item_description'])
 tfidf_description_train=tfidfvectorizer.transform(train_data['item_description'])
 tfidf_description_val=tfidfvectorizer.transform(validation_data['item_description'])
 tfidf_description_test=tfidfvectorizer.transform(test_data['item_description'])
 print("After Vectorization of item description feature: ")
 print(tfidf_description_train.shape)
 print(tfidf_description_val.shape)
 print(tfidf_description_test.shape)
 print("Some Features are: ")
 print(tfidfvectorizer.get_feature_names() [222:234] )<categorify>
plt.imshow(X_test[0].reshape(28,28), cmap='gray')
 y_predict = model.predict(X_test[0].reshape(1,28,28,1))
print('Value: ', np.argmax(y_predict))
features_train = csr_matrix(pd.get_dummies(train_data[['item_condition_id', 'shipping']],sparse=True ).values)
 features_val = csr_matrix(pd.get_dummies(validation_data[['item_condition_id', 'shipping']],sparse=True ).values)
 features_test = csr_matrix(pd.get_dummies(test_data[['item_condition_id', 'shipping']],sparse=True ).values)
 print(features_train.shape)
 print(features_val.shape)
 print(features_test.shape )<compute_train_metric>
%%time
y_pred = model.predict(sub )
linearregression=LinearRegression(normalize=True)
 linearregression.fit(X_train,train_data['log_prices'])
 ytrain_predict=linearregression.predict(X_train)
 yval_predict=linearregression.predict(X_val)
 train_error=np.sqrt(mean_squared_log_error(train_data['log_prices'],ytrain_predict))
val_error=np.sqrt(mean_squared_log_error(validation_data['log_prices'],yval_predict))
print(" Linear Regression RMSLE on train is {} RMSLE on cv is {}".format(train_error,val_error)) 
<predict_on_test>
y_sub = np.argmax(y_pred, 1 )
yval_linear=linearregression.predict(X_val)
 ytest_linear=linearregression.predict(X_test )<load_from_csv>
submission = pd.DataFrame({'ImageId': np.arange(1, 28001), 'Label': y_sub})
 submission.to_csv("submission.csv", index = False)
 print("Your submission was successfully saved!" )
submission_data = pd.read_csv('sample_submission_stg2.csv')
 submission_data.head(5 )<feature_engineering>
model.save("my_model" )