Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
df_subs['25'] = avg_bleand_1['TARGET'] 
<save_to_csv>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 sub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )
avg_bleand_2 = avg_bleand_1.copy()
avg_bleand_2['TARGET'] = 1.0*(7*df_subs['22'] + 12*df_subs['23'] + 16*df_subs['24'] - 5*df_subs['25'])/ 31
avg_bleand_2.to_csv('submission_final.csv', index=False )<import_modules>
X = train.drop(['label'], 1 ).values
y = train['label'].values
test_x = test.values
print(X)
 np.shape(X )
import re
import string
import os
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from nltk.corpus import stopwords
from tqdm import tqdm
import spacy
import random
from spacy.util import compounding
from spacy.util import minibatch<import_modules>
X = X / 255.0
test_x = test_x / 255.0
X = X.reshape(-1,28,28,1)
 test_x = test_x.reshape(-1,28,28,1 )
print('TF version',tf.__version__ )<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0 )
def load_data_of_train() :
train=pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 train['text'] = train['text'].astype(str)
 train['selected_text'] = train['selected_text'].astype(str)
 return train
def load_data_of_test() :
test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 return test
def load_data_of_submission() :
sub = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
 return sub<train_model>
mean = np.mean(X_train)
 std = np.std(X_train)
 def standardize(x):
return(x-mean)/std
train_data= load_data_of_train()
train_data.head()<count_values>
batch_size = 64
train_data.sentiment.value_counts()<correct_missing_values>
model=Sequential()
model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu", input_shape=(28,28,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=256, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(512,activation="relu"))
model.add(Dense(10,activation="softmax"))
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"] )
train_data.dropna(inplace=True )<split>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 train_gen = datagen.flow(X_train, y_train, batch_size=batch_size)
 test_gen = datagen.flow(X_test, y_test, batch_size=batch_size )
test_data = load_data_of_test()
sample_submission_data = load_data_of_submission()<split>
epochs = 50
history = model.fit_generator(train_gen,
epochs = epochs,
steps_per_epoch = X_train.shape[0] // batch_size,
validation_data = test_gen,
validation_steps = X_test.shape[0] // batch_size )
def jaccard(text1, text2):
a = set(text1.split())
 b = set(text2.split())
 intresection = a.intersection(b)
 IOU =(float )(len(intresection)) /(len(a)+len(b)-len(intresection))
return IOU<concatenate>
y_pred = model.predict(X_test)
 X_test__ = X_test.reshape(X_test.shape[0], 28, 28)
 fig, axis = plt.subplots(4, 4, figsize=(12, 14))
for i, ax in enumerate(axis.flat):
ax.imshow(X_test__[i], cmap='binary')
 ax.set(title = f"Real Number is {y_test[i].argmax() }
Predict Number is {y_pred[i].argmax() }");
jaccard_list=[]
def calc_jaccard() :
for row in train_data.itertuples() :
jaccard_list.append(jaccard(row.text, row.selected_text))
return jaccard_list;<feature_engineering>
pred = model.predict_classes(test_x, verbose=1 )
jac = calc_jaccard()
train_data['jaccard'] = jac<feature_engineering>
sub['Label'] = pred
sub.to_csv("DigitSolutionCNN.csv", index=False)
 sub.head()