Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_search_space>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
param_grid = {
'n_estimators': [200, 500],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [4,5,6,7,8],
'criterion' :['gini', 'entropy']
} 
<train_model>
df=pd.read_csv(".. /input/digit-recognizer/train.csv")
 df.head()
rfc1=RandomForestClassifier(random_state=42, max_features='log2', n_estimators= 200, max_depth=6, criterion='entropy')
 rfc1.fit(X_train, y_train )<save_to_csv>
df_test=pd.read_csv(".. /input/digit-recognizer/test.csv" ).values
df_test[0:5]
predictions = rfc1.predict(test_data[features])
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
X=df.drop(["label"],axis="columns" ).values
Y=df["label"].values
X=X/255.0
from typing import Tuple, Dict, Any
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.metrics import Mean
import tensorflow.keras.backend as K
import tensorflow.keras.layers<feature_engineering>
x_train,x_test,y_train,y_test=train_test_split(X,Y,test_size=0.2 )
def extract_titanic_features(
 source: pd.DataFrame, 
):

 features = {
name: np.array(value, np.float32)
 for name, value in source[["Age", "Fare", "SibSp", "Parch", "Pclass"]].items()
}
for age in [4, 8, 12, 16, 22, 26, 30, 35, 40, 45, 50, 60]:
features['at_least_' + str(age)] = np.array(source['Age'] >= age, np.float32)
 features['name_len'] = np.array(source["Name"].str.len() , np.float32)
 for title in ['Miss.', 'Master', 'Mr.', 'Mrs.']:
features['is_' + title] = np.array(source['Name'].str.contains(title), np.float32)
 features["Sex"] = np.array(source["Sex"], np.str)
 features["Embarked"] = np.array(source["Embarked"], np.str)
 return features
train_labels = titanic_train_df["Survived"]
train_features = extract_titanic_features(titanic_train_df)
 test_features = extract_titanic_features(titanic_test_df)
 inputs = {
k: tf.keras.Input(shape=(1,), name=k, dtype=v.dtype)
 for k, v in train_features.items()
}
preprocessed_inputs = []
numeric_inputs = {k: v for k, v in inputs.items() if v.dtype == tf.float32}
def nan_mask(
 layer: tf.keras.layers.Layer, 
)-> Tuple[tf.keras.layers.Layer, tf.keras.layers.Layer]:

 is_nan = tf.math.is_nan(layer)
 finite_input = tf.where(is_nan, tf.zeros_like(layer), layer)
 norm_input = tf.keras.layers.BatchNormalization()(finite_input)
 mask = tf.cast(tf.math.logical_not(is_nan), tf.float32)
 return mask, norm_input
finite_numeric_inputs = {}
numeric_masks = {}
for k, v in numeric_inputs.items() :
mask_v, finite_v = nan_mask(v)
 preprocessed_inputs.append(mask_v)
 preprocessed_inputs.append(finite_v)
 finite_numeric_inputs[k] = finite_v
numeric_masks[k] = mask_v
accomp_count = tf.math.add(
 finite_numeric_inputs["Parch"], finite_numeric_inputs["SibSp"] 
)
 preprocessed_inputs.append(accomp_count)
 gender_vocab = np.unique(train_features["Sex"])
 gender_lookup = tf.keras.layers.experimental.preprocessing.StringLookup(
 vocabulary=gender_vocab 
)
 gender_onehot = tf.keras.layers.experimental.preprocessing.CategoryEncoding(
 max_tokens=gender_lookup.vocab_size() 
)
 gender_input = gender_onehot(gender_lookup(inputs["Sex"]))
preprocessed_inputs.append(gender_input)
 embarked_vocab = np.array(( "C", "Q", "S"))
embarked_lookup = tf.keras.layers.experimental.preprocessing.StringLookup(
 vocabulary=embarked_vocab 
)
 embarked_onehot = tf.keras.layers.experimental.preprocessing.CategoryEncoding(
 max_tokens=embarked_lookup.vocab_size() 
)
 embarked_input = embarked_onehot(embarked_lookup(inputs["Embarked"]))
preprocessed_inputs.append(embarked_input)
 x = tf.keras.layers.Concatenate()(preprocessed_inputs)
 preprocess_model = tf.keras.Model(inputs, x)
 tf.keras.utils.plot_model(
 preprocess_model,
to_file="preprocess_model.png",
rankdir="LR",
show_shapes=True,
show_layer_names=True,
)<choose_model_class>
X=X.reshape(len(X),28,28,-1)
 x_train,x_test,y_train,y_test=train_test_split(X,Y,test_size=0.2 )
head = preprocess_model(inputs)
 columns = []
activation="relu"
for i in range(4):
c = head
c = tf.keras.layers.Dense(64, activation=activation )(c)
 c = tf.keras.layers.Dropout(0.5 )(c)
 c = tf.keras.layers.Dense(64, activation=activation )(c)
 c = tf.keras.layers.BatchNormalization()(c)
 c = tf.keras.layers.Dropout(0.2 )(c)
 c = tf.keras.layers.Dense(32, activation=activation )(c)
 columns.append(c)
 if len(columns)== 1:
tail = columns[0]
else:
tail = tf.keras.layers.Add()(columns)
 tail = tf.keras.layers.Dense(16, activation="relu" )(tail)
 tail = tf.keras.layers.Dense(1, activation="sigmoid" )(tail)
 model = tf.keras.Model(inputs, tail)
 model.compile(
 loss=tf.losses.BinaryCrossentropy() ,
optimizer=tfa.optimizers.Lookahead(tf.keras.optimizers.Nadam()),
metrics=["accuracy"], 
)
 if False:
model.summary()
tf.keras.utils.plot_model(
 model,
to_file="final_model.png",
rankdir="LR",
show_shapes=True, 
)
 callbacks = [
tf.keras.callbacks.ProgbarLogger() ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
patience=10, 
),
]
fit_params = dict(
 batch_size=128,
validation_freq=1,
validation_split=0.1,
epochs=300,
verbose=0,
callbacks=[callbacks], 
)
 submit = True
if submit:
fit_params.update(dict(
 validation_freq=0,
validation_split=0,
epochs=125,
verbose=0, 
))
history = model.fit(
 train_features,
train_labels.to_numpy() ,
**fit_params, 
)
 print("accuracy:", history.history['accuracy'][-1])
 if not submit:
print("val_accuracy:", history.history['val_accuracy'][-1])
 plt.plot(history.history['accuracy'][10:])
 plt.plot(history.history['loss'][10:])
 if not submit:
plt.plot(history.history['val_accuracy'][10:])
 plt.plot(history.history['val_loss'][10:])
 plt.xlabel('epoch')
 plt.show()<save_to_csv>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(x_train )
test_predictions = model(test_features)
 test_ids = titanic_test_df["PassengerId"].to_numpy()
test_hard_predictions =(
 np.floor(np.array(test_predictions)+ 0.5 ).astype("int" ).reshape(-1)
)
 pred_df = pd.Series(data=test_hard_predictions, name="Survived", index=test_ids)
 pred_df.to_csv(
 "submission.csv",
index_label="PassengerId",
header=["Survived"],
)<load_from_csv>
reduce_lr = tf.keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train_data.info()<sort_values>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(16,(3,3),padding='same',activation='relu',input_shape=(28,28,1)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(32,(3,3),padding='same',activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(64,(3,3),padding='same',activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(64,(3,3),padding='same',activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(1024, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(512, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation="softmax")
 ])
 optimizer=tf.keras.optimizers.Adam(lr=0.001)
 model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 model.summary()
missing_train_total = train_data.isnull().sum().sort_values(ascending= False)
 missing_train_percentage =(train_data.isnull().sum() /train_data.count() ).sort_values(ascending= False)
 missing_train_data = pd.concat([missing_train_total, missing_train_percentage], axis=1, keys=['Total', 'Percent'])
 missing_train_data.head(10 )<sort_values>
batch_size = 64
epochs = 20
history = model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size), epochs = epochs,
validation_data =(x_test, y_test), verbose=2,
steps_per_epoch=x_train.shape[0] // batch_size,
callbacks = [reduce_lr] )
missing_test_total = test_data.isnull().sum().sort_values(ascending= False)
 missing_test_percentage =(test_data.isnull().sum() /test_data.count() ).sort_values(ascending= False)
 missing_test_data = pd.concat([missing_test_total, missing_test_percentage], axis=1, keys=['Total', 'Percent'])
 missing_test_data.head(10 )<groupby>
model.evaluate(x_test,y_test )
train_data.groupby('Pclass')['Age'].mean()<groupby>
y_pred=model.predict(x_test)
 y_preds=[np.argmax(pred)for pred in y_pred]
accuracy_score(y_preds,y_test )
train_data.groupby(['Pclass','Sex'])['Sex'].count()<groupby>
df_test=df_test.reshape(len(df_test),28,28,-1)
 a_pred=model.predict(( df_test)/255.0)
 a_preds=[np.argmax(pred)for pred in a_pred]
samp=pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 samp["Label"]=a_preds
samp.head()
train_data.groupby('Sex')['Sex'].count()<groupby>
samp.to_csv("outputCNN.csv",index=False )
mean = train_data.groupby(['Pclass','Sex'])['Age'].mean()
median = train_data.groupby(['Pclass','Sex'])['Age'].median()
age_sex_Pclass = pd.concat([mean, median], axis=1, keys=['Age mean', 'Age median'])
 age_sex_Pclass.head(6)
<count_values>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
train_data.Embarked.value_counts()<prepare_x_and_y>
df=pd.read_csv(".. /input/digit-recognizer/train.csv")
 df.head()
features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']
X = train_data[features]
y = train_data.Survived
Pclass_Sex_Age_median = X.groupby(['Pclass','Sex'] ).Age.transform('median')
 X.Age.fillna(Pclass_Sex_Age_median, inplace = True)
 Pclass_Fare_median = X.groupby('Pclass' ).Fare.transform('median')
 X.Fare.fillna(Pclass_Fare_median, inplace = True)
 missing_test_total = X.isnull().sum().sort_values(ascending= False)
 X.Embarked.fillna('S', inplace = True)
 X['Embarked'].replace({'S': 0, 'Q': 1, 'C': 2}, inplace = True)
 X['Sex'].replace({'female': 0, 'male': 1}, inplace = True)
 X.info()
print(X.Sex )<split>
df_test=pd.read_csv(".. /input/digit-recognizer/test.csv" ).values
df_test[0:5]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.25, random_state=1)
 print(type(X_train)) 
<predict_on_test>
X=df.drop(["label"],axis="columns" ).values
Y=df["label"].values
X=X/255.0
clf_mae=[]
for md in range(8):
clf = RandomForestClassifier(max_depth=(md+1), random_state=1)
 clf.fit(X_train, y_train)
 predictions = clf.predict(X_val)
 mae =(1.- mean_absolute_error(predictions, y_val)) *100.
 clf_mae.append(mae)
 print(clf_mae )<predict_on_test>
x_train,x_test,y_train,y_test=train_test_split(X,Y,test_size=0.2 )
clf = RandomForestClassifier(max_depth=4, random_state=1)
 clf.fit(X_train, y_train)
 predictions = clf.predict(X_val)
 mae =(1.- mean_absolute_error(predictions, y_val)) *100.
 print(mae )<compute_train_metric>
X=X.reshape(len(X),28,28,-1)
 x_train,x_test,y_train,y_test=train_test_split(X,Y,test_size=0.2 )
logreg = LogisticRegression(solver='liblinear')
 logreg.fit(X_train, y_train)
 predictions = logreg.predict(X_val)
 mae = mean_absolute_error(predictions, y_val)
 print(( 1-mae)*100,"%")
<sort_values>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(x_train )
missing_test_total = test_data.isnull().sum().sort_values(ascending= False)
 missing_test_total.head()<categorify>
reduce_lr = tf.keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )
Pclass_Sex_Age_median = test_data.groupby(['Pclass','Sex'] ).Age.transform('median')
 test_data.Age.fillna(Pclass_Sex_Age_median, inplace = True)
 Pclass_Fare_median = test_data.groupby('Pclass' ).Fare.transform('median')
 test_data.Fare.fillna(Pclass_Fare_median, inplace = True)
 test_data['Embarked'].replace({'S': 0, 'Q': 1, 'C': 2}, inplace = True)
 test_data['Sex'].replace({'female': 0, 'male': 1}, inplace = True)
<sort_values>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(16,(3,3),padding='same',activation='relu',input_shape=(28,28,1)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(32,(3,3),padding='same',activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(64,(3,3),padding='same',activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(64,(3,3),padding='same',activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(1024, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(512, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation="softmax")
 ])
 optimizer=tf.keras.optimizers.Adam(lr=0.001)
 model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 model.summary()
missing_test_total = test_data.isnull().sum().sort_values(ascending= False)
 missing_test_total.head()<save_to_csv>
batch_size = 64
epochs = 20
history = model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size), epochs = epochs,
validation_data =(x_test, y_test), verbose=2,
steps_per_epoch=x_train.shape[0] // batch_size,
callbacks = [reduce_lr] )
<data_type_conversions>
model.evaluate(x_test,y_test )
name_list_train = train_data.Name.to_list()
print(name_list_train )<count_values>
y_pred=model.predict(x_test)
 y_preds=[np.argmax(pred)for pred in y_pred]
accuracy_score(y_preds,y_test )
list_of_titles = title_retriever(name_list_train)
 print(len(list_of_titles))
train_data['Title'] = list_of_titles
train_data.Title.value_counts()<categorify>
df_test=df_test.reshape(len(df_test),28,28,-1)
 a_pred=model.predict(( df_test)/255.0)
 a_preds=[np.argmax(pred)for pred in a_pred]
samp=pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 samp["Label"]=a_preds
samp.head()
Title_Age_median = train_data.groupby(['Title'] ).Age.transform('median')
 train_data.Age.fillna(Title_Age_median, inplace = True)
<data_type_conversions>
samp.to_csv("outputCNN.csv",index=False )