Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<groupby>
%matplotlib inline
deck_class_surv_count = train[['Deck', 'Pclass', 'Survived']]
deck_class_surv_count = deck_class_surv_count.groupby(['Deck', 'Pclass'] ).sum()
deck_class_surv_count, _ = get_deck_class_count(deck_class_surv_count, False)
 deck_class_surv_prop = deck_class_surv_count.copy()
for deck in DECKS:
for pclass in deck_class_count[deck].keys() :
try:
deck_class_surv_prop[deck][pclass] = round(( deck_class_surv_count[deck][pclass] /
deck_class_count[deck][pclass]), 2)
 except ZeroDivisionError:
pass
print('Decimal percent of passengers survived for each 'Deck', 'Pclass' combination(training set)':')
 display(deck_class_surv_prop )<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
combined = combine_df(train, test)
 combined['DeckPclassSurvProp'] = combined.apply(lambda x: get_surv_prop(x['Deck'], x['Pclass']),axis=1 )<split>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
combined['FamilySize'] = combined['SibSp'] + combined['Parch'] + 1
COLS_TO_DROP.extend(['SibSp', 'Parch'])
 train, test = divide_df(combined, TRAIN_LEN )<count_values>
y_train = train['label']
x_train = train.drop('label', axis=1)
 y_train.shape, x_train.shape
combined = combine_df(train, test)
 combined['Title'] = combined['Name'].apply(lambda name: name.split(',')[1].split('.')[0].strip())
 print('Count of passenger titles aboard the Titanic:')
 combined['Title'].value_counts()<define_variables>
y_train.value_counts()
normalized_titles = {
"Capt": "Officer",
"Col": "Officer",
"Don": "Royalty",
"Dona": "Royalty",
"Dr": "Officer",
"Jonkheer": "Royalty",
"Lady" : "Royalty",
"Major": "Officer",
"Master" : "Master",
"Miss" : "Miss",
"Mlle": "Miss",
"Mme": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Ms": "Mrs",
"Rev": "Officer",
"Sir" : "Royalty",
"the Countess": "Royalty"}
combined['Title'] = combined['Title'].map(normalized_titles)
 print('Count of updated passenger titles aboard the Titanic:')
 combined['Title'].value_counts()<sort_values>
x_train = x_train/255.0
test = test/255.0
print('Decimal percentages for survival based on passenger title:')
 combined[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean() \ 
.sort_values(by='Survived', kind="quicksort", ascending=False )<feature_engineering>
y_train = pd.DataFrame(data=y_train)
 one_hot = OneHotEncoder(handle_unknown='ignore')
 one_hot.fit(y_train.values)
 y_train = one_hot.transform(y_train.values ).toarray()
combined['Surname'] = combined['Name'].apply(lambda x: str.split(x, ",")[0] )<filter>
random_seed = 3
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.1, random_state=random_seed)
 x_train.shape, x_val.shape, y_train.shape, y_val.shape
display(combined.loc[combined['Surname'] == 'Davies'] )<filter>
model = Sequential([
Conv2D(filters = 64, input_shape=(28,28,1), kernel_size=(3,3), strides=(1,1), padding='valid'),
Activation('relu'),
MaxPooling2D(pool_size=(2,2), strides=(1,1), padding='valid'),
BatchNormalization() ,
Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding='valid'),
Activation('relu'),
MaxPooling2D(pool_size=(2,2), strides=(1,1), padding='valid'),
BatchNormalization() ,
Conv2D(filters=192, kernel_size=(1,1), strides=(1,1), padding='valid'),
Activation('relu'),
BatchNormalization() ,
Conv2D(filters=192, kernel_size=(3,3), strides=(1,1), padding='valid'),
Activation('relu'),
BatchNormalization() ,
Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding='valid'),
Activation('relu'),
MaxPooling2D(pool_size=(2,2), strides=(1,1), padding='valid'),
BatchNormalization() ,
Flatten() ,
Dense(2048),
Activation('relu'),
Dropout(0.4),
BatchNormalization() ,
Dense(2048),
Activation('relu'),
Dropout(0.4),
BatchNormalization() ,
Dense(800),
Activation('relu'),
Dropout(0.4),
BatchNormalization() ,
Dense(10),
Activation('softmax'),
] )
combined.loc[combined['FamilySize'] == 11]<groupby>
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'] )
combined = group_survivors(combined, ['Surname', 'Fare', 'FamilySize'], 'FamilySurvival')
 print('Count of passengers with family survival data: ',
combined.loc[combined['FamilySurvival']!=0.5].shape[0] )<groupby>
model.fit(x_train, y_train, batch_size=200, validation_data=(x_val,y_val), epochs = 10 )
combined = group_survivors(combined, ['Ticket'], 'GroupSurvival')
 print('Count of passenger with group survival data: ',
combined[combined['GroupSurvival']!=0.5].shape[0] )<categorify>
results = model.predict(test )
<categorify><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_submission1.csv",index=False )