Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
warnings.filterwarnings('ignore')
 %matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams[ 'figure.figsize' ] = 8 , 6
pd.set_option('max_columns', 90)
 PALETTE = ['
BACKCOLOR = '
sns.set_palette(PALETTE)
 warnings.filterwarnings('ignore')
 mpl.rcParams['figure.dpi'] = 120
mpl.rcParams['axes.spines.top'] = False
mpl.rcParams['axes.spines.right'] = False<load_from_csv>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 all_data = pd.concat(( train, test)).reset_index(drop=True )<categorify>
df = train.copy()
def multi_table(table_list):
return HTML(
 f"<table><tr> {''.join(['<td>' + table._repr_html_() + '</td>' for table in table_list])} </tr></table>" )<count_values>
y = df.label.values.astype('int32')
 df = df[df.columns[1:]].values.astype('float32')
 X_train , X_test , y_train , y_test = train_test_split(df , y , test_size = 0.2 , random_state = 100 )
multi_table([pd.DataFrame(all_data[i].value_counts())for i in all_data.columns] )<define_variables>
test = test.values.astype('float32' )
numerical_vars = ['Age', 'SibSp', 'Parch', 'Fare']
ordinal_vars = ['Pclass']
nominal_vars = ['Survived', 'Name', 'Sex', 'Ticket', 'Cabin', 'Embarked']<create_dataframe>
new = pd.read_csv('.. /input/digit-recognizer/train.csv' )
train0 = train[train.Survived == 0]
train1 = train[train.Survived == 1]
cnt = 0
detail_desc = []
for c in train.columns:
if c == 'PassengerId':
continue
if train[c].dtypes != 'object':
desc = pd.DataFrame(columns=['feature', 'data', 'type', 'count', 'mean', 'median', 'std', 'min', 'max', 'skew', 'null'])
 desc.loc[0] = [c, 'Train', train[c].dtype.name, train[c].count() , train[c].mean() , train[c].median() , train[c].std() , train[c].min() , train[c].max() , train[c].skew() , train[c].isnull().sum() ]
desc.loc[1] = [c, 'All', train[c].dtype.name, all_data[c].count() , all_data[c].mean() , all_data[c].median() , all_data[c].std() , all_data[c].min() , all_data[c].max() , all_data[c].skew() , all_data[c].isnull().sum() ]
desc.loc[2] = [c, 'Target=0', train0[c].dtype.name, train0[c].count() , train0[c].mean() , train0[c].median() , train0[c].std() , train0[c].min() , train0[c].max() , train0[c].skew() , train0[c].isnull().sum() ]
desc.loc[3] = [c, 'Target=1', train1[c].dtype.name, train1[c].count() , train1[c].mean() , train1[c].median() , train1[c].std() , train1[c].min() , train1[c].max() , train1[c].skew() , train1[c].isnull().sum() ]
desc = desc.set_index(['feature', 'data'],drop=True)
 detail_desc.append(desc.style.background_gradient() )<create_dataframe>
label_counter = new.label.value_counts()
print(label_counter )
train0 = train[train.Survived == 0]
train1 = train[train.Survived == 1]
cnt = 0
detail_desc = []
for c in train.columns:
if c == 'PassengerId':
continue
if train[c].dtypes == 'object':
desc = pd.DataFrame(columns=['feature', 'data', 'type', 'count', 'null', 'mode', 'value_count'])
 desc.loc[0] = [c, 'Train', train[c].dtype.name, train[c].count() , train[c].isnull().sum() , train[c].mode() , train[c].value_counts() ]
desc = desc.set_index(['feature', 'data'],drop=True)
 detail_desc.append(desc.style.background_gradient() )<create_dataframe>
X_train = X_train / 255
X_test = X_test / 255
test = test / 255
multi_table(detail_desc )<feature_engineering>
y_train = keras.utils.to_categorical(y_train, 10)
 y_test = keras.utils.to_categorical(y_test, 10 )
tmp_train = copy.deepcopy(train)
 tmp_train['AgeBin'] = 6
for i in range(6):
tmp_train.loc[(tmp_train.Age >= 10*i)&(tmp_train.Age < 10*(i + 1)) , 'AgeBin'] = i
tmp_train.head(3 )<concatenate>
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
cat_dist(tmp_train, var='AgeBin', hue='Survived', msg_show=False )<categorify>
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='relu', padding='same',input_shape=(28 , 28 , 1)))
 model.add(Conv2D(32,(3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(64,(3, 3), activation='relu', padding='same'))
model.add(Conv2D(64,(3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(100, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
tmp_train = copy.deepcopy(train)
 tmp_train['FareBin'] = pd.cut(tmp_train.Fare, 10)
 tmp_train['FareBin'] = LabelEncoder().fit_transform(tmp_train.FareBin)
 tmp_train.head(3 )<concatenate>
model.compile(optimizer = RMSprop(lr=0.001),loss='categorical_crossentropy', metrics=['accuracy'] )
cat_dist(tmp_train, var='FareBin', hue='Survived', msg_show=False )<concatenate>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.0,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
cat_dist(train, var='Embarked', hue='Survived' )<create_dataframe>
batch_size = 64
epochs = 10
lr_reduce = ReduceLROnPlateau(monitor='val_acc', factor=0.1, epsilon=0.0001, patience=1, verbose=1 )
tmp_all_data = copy.deepcopy(all_data)
 t0 = pd.DataFrame(tmp_all_data.Name)
 t1 = pd.DataFrame(tmp_all_data.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip() ).value_counts())
 multi_table([t0, t1] )<feature_engineering>
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[lr_reduce],
validation_data=(X_test, y_test),
epochs = epochs, verbose = 2 )
tmp_all_data['Title'] = tmp_all_data.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
<count_values>
score = model.evaluate(X_test, y_test, verbose=0)
 print('valid loss:', score[0])
 print('valid accuracy:', score[1] )
tmp_train.Cabin.value_counts()<feature_engineering>
pred = model.predict(test )
tmp_train['CabinCnt'] = tmp_train.Cabin.apply(lambda x: 0 if pd.isna(x)else len(x.split(' ')))
 tmp_train['CabinClass'] = tmp_train.Cabin.apply(lambda x: str(x)[0] )<create_dataframe>
pred_digits = np.argmax(pred , axis = 1)
 ImageId = range(1 , len(pred_digits)+1 )
t0 = pd.DataFrame(tmp_train.CabinCnt.value_counts())
 t1 = pd.DataFrame(tmp_train.CabinClass.value_counts())
 multi_table([t0, t1] )<concatenate>
context = {"ImageId" : ImageId , "Label" : pred_digits }
ans = pd.DataFrame(context )
cat_dist(tmp_train, var='CabinCnt', hue='Survived', msg_show=False )<concatenate>
ans.to_csv("Predictions by CNN.csv", index=None )