Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
pd.set_option("display.precision", 4)
 sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 TRAIN_LEN = 891
RNG_SEED = 343
COLS_TO_DROP = []
CHILD_AGE_END = 18
DECKS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'N']
DEFAULT_SURVIVAL = 0.5
def get_deck_class_count(df, T_deck):

 deck_count = {'A': {}, 'B': {}, 'C': {}, 'D': {}, 'E': {}, 'F': {},
'G': {}, 'N': {}, 'T': {}}
if not T_deck:
deck_count.pop('T', None)
 deck_percent = {}
decks = df.transpose().columns.levels[0]
for deck in decks:
for pclass in range(1, 4):
try:
count = int(df.loc[deck, pclass])
 deck_count[deck][pclass] = count
except KeyError:
deck_count[deck][pclass] = 0
deck_percent[deck] = [(count / sum(deck_count[deck].values()))
* 100 for count in deck_count[deck].values() ]
return deck_count, deck_percent
def get_surv_prop(deck, pclass):

 return deck_class_surv_prop[deck][pclass]
def get_corr(df):

 corr = df.corr().abs().unstack().sort_values(kind="quicksort",
ascending=False ).reset_index()
cols_map = {"level_0": "Feature 1",
"level_1": "Feature 2",
0: 'Correlation Coefficient'}
corr.drop(corr.iloc[1::2].index, inplace=True)
 corr.rename(columns=cols_map, inplace=True)
 return corr.drop(corr[corr['Correlation Coefficient'] == 1.0].index)
 def combine_df(df1, df2):

 return pd.concat([df1, df2], sort=True ).reset_index(drop=True)
 def divide_df(df, first_len):

 return df.loc[:first_len - 1], df.loc[first_len:].drop(['Survived'], axis=1)
 def drop_cols(cols):

 for col in cols:
train.drop([col], inplace=True, axis=1)
 test.drop([col], inplace=True, axis=1)
 return
def display_class_dist(percentages, y_label, title):

 df_percent = pd.DataFrame(percentages ).transpose()
deck_names = percentages.keys()
bar_count = np.arange(len(deck_names))
bar_width = 0.75
plt.figure(figsize=(16, 8))
plt.bar(bar_count, df_percent[0],
color='red', edgecolor='black', width=bar_width,
label='Passenger Class 1')
 plt.bar(bar_count, df_percent[1], bottom=df_percent[0],
color='lime', edgecolor='black', width=bar_width,
label='Passenger Class 2')
 plt.bar(bar_count, df_percent[2], bottom=df_percent[0] + df_percent[1],
color='blue', edgecolor='black', width=bar_width,
label='Passenger Class 3')
 plt.xlabel('Deck', size=25)
 plt.ylabel(y_label, size=25)
 plt.xticks(bar_count, deck_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper right', prop={'size': 15})
 plt.title(title, size=30, y=1)
 plt.show()
return
def group_survivors(df, group, new_feature_name):

 df[new_feature_name] = DEFAULT_SURVIVAL
for _, group_df in df.groupby(group):
if len(group_df)> 1:
surv_max = group_df['Survived'].max()
surv_min = group_df['Survived'].min()
if isnan(surv_max)and isnan(surv_min):
continue
for _, row in group_df.iterrows() :
passId = row['PassengerId']
if(surv_max == 1.0):
df.loc[df['PassengerId'] == passId, new_feature_name] = 1.0
elif(surv_min==0.0):
df.loc[df['PassengerId'] == passId, new_feature_name] = 0.0
return df<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 combined = combine_df(train, test )<sort_values>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<filter>
Y_train = train['label']
adults = train[train['Age'] >= CHILD_AGE_END]
children = train[train['Age'] < CHILD_AGE_END]
print('Proportion of passengers <{} who survived: {:.4f}'.format(CHILD_AGE_END, children['Survived'].mean()))
print('Proportion of passengers >={} who survived: {:.4f}'.format(CHILD_AGE_END, adults['Survived'].mean()))<sort_values>
X_train = train.drop(labels=['label'], axis =1)
 del train
train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<filter>
X_train = X_train/255.0
test = test/255.0
combined[combined['Fare'].isnull() ]<data_type_conversions>
Y_train = to_categorical(Y_train , num_classes = 10 )
combined['Fare'] = combined['Fare'].fillna(med_fare[3][0][0] )<filter>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state = 42 )
combined[combined['Embarked'].isnull() ]<data_type_conversions>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu'))
model.add(MaxPool2D(pool_size =(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu'))
model.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation = 'relu'))
model.add(MaxPool2D(pool_size =(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = 'relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation = 'softmax'))
combined['Embarked'] = combined['Embarked'].fillna('S' )<split>
model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'] )
train, test = divide_df(combined, TRAIN_LEN)
 corr_train = get_corr(train)
 corr_train[(corr_train['Feature 1'] == 'Age')| 
(corr_train['Feature 2'] == 'Age')]<groupby>
epochs = 30
batch_size = 86
age_pclass_sex = train.groupby(['Pclass', 'Sex'] ).median() ['Age']
print('Median ages for the following groups(training data):')
 for pclass in range(1, train['Pclass'].nunique() + 1):
for sex in ['female', 'male']:
print('Pclass {} {}s: {}'.format(pclass, sex, age_pclass_sex[pclass][sex]))
print('All passengers: {}'.format(train['Age'].median()))<categorify>
datagen = ImageDataGenerator(rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1 
)
 datagen.fit(X_train)
combined['Age'] = combined.groupby(['Pclass', 'Sex'])['Age'].apply(lambda x: x.fillna(x.median()))<feature_engineering>
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size = batch_size), epochs = epochs, validation_data =(X_val, Y_val), steps_per_epoch=X_train.shape[0] // batch_size )
combined['Deck'] = combined['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'N')
 train, test = divide_df(combined, TRAIN_LEN)
 deck_class_count = train.groupby(['Deck', 'Pclass'] ).count().rename(columns={'Name': 'Count'})
 deck_class_count = deck_class_count[['Count']]
print('Passenger counts for each Deck, Pclass combination(training data)where N deck denotes null values:',
deck_class_count, sep='
' )<concatenate>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results, name = 'Label' )
<split><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )