Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<data_type_conversions>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Convolution2D, MaxPooling2D
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam,RMSprop
from keras.layers.normalization import BatchNormalization
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train = process_date(train)
 test = process_date(test )<groupby>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
d = train['release_date'].dt.year.value_counts().sort_index()
g = train.groupby('release_date')['revenue'].sum()<count_values>
y_train = train['label'].astype('int32')
 X_train =(train.drop(['label'], axis = 1)).values.astype('float32')
 X_test = test.values.astype('float32')
 batch_size, img_rows, img_cols = 64, 28, 28
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
 X_train.shape, X_test.shape
train['status'].value_counts()<merge>
X_train /= 255
X_test /= 255
rating_na = train.groupby(["release_year","original_language"])['rating'].mean().reset_index()
train[train.rating.isna() ]['rating'] = train.merge(rating_na, how = 'left' ,on = ["release_year","original_language"])
 vote_count_na = train.groupby(["release_year","original_language"])['totalVotes'].mean().reset_index()
train[train.totalVotes.isna() ]['totalVotes'] = train.merge(vote_count_na, how = 'left' ,on = ["release_year","original_language"])
 train['weightedRating'] =(train['rating']*train['totalVotes'] + 6.367 * 1000)/(train['totalVotes'] + 1000)
 train['inflationBudget'] = np.log1p(np.expm1(train['budget'])+ np.expm1(train['budget'])*1.8/100*(2018-train['release_year']))
train['_popularity_mean_year'] = train['popularity'] / train.groupby("release_year")["popularity"].transform('mean')
 train['_budget_runtime_ratio'] = train['budget']/train['runtime']
train['_budget_popularity_ratio'] = train['budget']/train['popularity']
train['_budget_year_ratio'] = train['budget']/(train['release_year']*train['release_year'])
 train['_releaseYear_popularity_ratio'] = train['release_year']/train['popularity']
train['_popularity_totalVotes_ratio'] = train['totalVotes']/train['popularity']
train['_rating_popularity_ratio'] = train['rating']/train['popularity']
train['_rating_totalVotes_ratio'] = train['totalVotes']/train['rating']
train['_totalVotes_releaseYear_ratio'] = train['totalVotes']/train['release_year']
train['_budget_rating_ratio'] = train['budget']/train['rating']
train['_runtime_rating_ratio'] = train['runtime']/train['rating']
train['_budget_totalVotes_ratio'] = train['budget']/train['totalVotes']
train['meanruntimeByYear'] = train.groupby("release_year")["runtime"].aggregate('mean')
 train['meanPopularityByYear'] = train.groupby("release_year")["popularity"].aggregate('mean')
 train['meanBudgetByYear'] = train.groupby("release_year")["budget"].aggregate('mean')
 train['meantotalVotesByYear'] = train.groupby("release_year")["totalVotes"].aggregate('mean')
 train['meanTotalVotesByRating'] = train.groupby("rating")["totalVotes"].aggregate('mean')
 train['isTaglineNA'] = 0
train.loc[train['tagline'] == 0 ,"isTaglineNA"] = 1
train['isTitleDifferent'] = 1
train.loc[ train['original_title'] == train['title'] ,"isTitleDifferent"] = 0 
<merge>
y_train = np_utils.to_categorical(y_train, 10 )
rating_na = test.groupby(["release_year","original_language"])['rating'].mean().reset_index()
test[test.rating.isna() ]['rating'] = test.merge(rating_na, how = 'left' ,on = ["release_year","original_language"])
 vote_count_na = test.groupby(["release_year","original_language"])['totalVotes'].mean().reset_index()
test[test.totalVotes.isna() ]['totalVotes'] = test.merge(vote_count_na, how = 'left' ,on = ["release_year","original_language"])
 test['weightedRating'] =(test['rating']*test['totalVotes'] + 6.367 * 1000)/(test['totalVotes'] + 1000)
 test['inflationBudget'] = np.log1p(np.expm1(test['budget'])+ np.expm1(test['budget'])*1.8/100*(2018-test['release_year']))
test['_popularity_mean_year'] = test['popularity'] / test.groupby("release_year")["popularity"].transform('mean')
 test['_budget_runtime_ratio'] = test['budget']/test['runtime']
test['_budget_popularity_ratio'] = test['budget']/test['popularity']
test['_budget_year_ratio'] = test['budget']/(test['release_year']*test['release_year'])
 test['_releaseYear_popularity_ratio'] = test['release_year']/train['popularity']
test['_popularity_totalVotes_ratio'] = test['totalVotes']/test['popularity']
test['_rating_popularity_ratio'] = test['rating']/test['popularity']
test['_rating_totalVotes_ratio'] = test['totalVotes']/test['rating']
test['_totalVotes_releaseYear_ratio'] = test['totalVotes']/test['release_year']
test['_budget_rating_ratio'] = test['budget']/test['rating']
test['_runtime_rating_ratio'] = test['runtime']/test['rating']
test['_budget_totalVotes_ratio'] = test['budget']/test['totalVotes']
test['meanruntimeByYear'] = test.groupby("release_year")["runtime"].aggregate('mean')
 test['meanPopularityByYear'] = test.groupby("release_year")["popularity"].aggregate('mean')
 test['meanBudgetByYear'] = test.groupby("release_year")["budget"].aggregate('mean')
 test['meantotalVotesByYear'] = test.groupby("release_year")["totalVotes"].aggregate('mean')
 test['meanTotalVotesByRating'] = test.groupby("rating")["totalVotes"].aggregate('mean')
 test['isTaglineNA'] = 0
test.loc[test['tagline'] == 0 ,"isTaglineNA"] = 1
test['isTitleDifferent'] = 1
test.loc[ test['original_title'] == test['title'] ,"isTitleDifferent"] = 0 
<drop_column>
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train,
test_size = 0.1, random_state = 12345 )
train = train.drop(['id','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline','all_genres',
'all_prod_companies','all_prod_countries','all_langs','all_keywords','all_cast_name'],axis=1)
 test = test.drop(['id','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline','all_genres',
'all_prod_companies','all_prod_countries','all_langs','all_keywords','all_cast_name'],axis=1 )<train_model>
input_shape =(img_rows, img_cols, 1)
 callback_es = EarlyStopping(monitor = 'val_accuracy', patience = 3)
 def first_cnn_model_keras(optimizer):
model = Sequential()
model.add(Convolution2D(64, 5, 5, padding = 'same', kernel_initializer = 'he_uniform',
input_shape = input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2), padding = 'same'))
model.add(Convolution2D(128, 5, 5, padding = 'same', kernel_initializer = 'he_uniform'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2), padding = 'same'))
model.add(Flatten())
 model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(optimizer, loss='categorical_crossentropy', metrics = ['accuracy'])
 return model
train.fillna(value=0.0, inplace = True)
 test.fillna(value=0.0, inplace = True )<data_type_conversions>
model1 = first_cnn_model_keras(Adam(learning_rate = 0.001, amsgrad = True))
h1 = model1.fit(X_train, y_train, batch_size = batch_size, epochs = 20, verbose = 1,
validation_data =(X_valid, y_valid), callbacks = [callback_es])
 final_loss_first_adam, final_acc_first_adam = model1.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_first_adam, final_acc_first_adam))
def clean_dataset(df):
assert isinstance(df, pd.DataFrame), "df needs to be a pd.DataFrame"
df.dropna(inplace=True)
 indices_to_keep = ~df.isin([np.nan, np.inf, -np.inf] ).any(1)
 return df[indices_to_keep].astype(np.float64)
 train = clean_dataset(train )<split>
model2 = first_cnn_model_keras(RMSprop(lr=0.001))
h2 = model2.fit(X_train, y_train, batch_size = batch_size, epochs = 20, verbose = 1,
validation_data =(X_valid, y_valid),callbacks = [callback_es])
 final_loss_first_rmsprop, final_acc_first_rmsprop = model2.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_first_rmsprop, final_acc_first_rmsprop))
X = train.drop(['revenue'],axis=1)
 y = train.revenue
X_train, X_valid, y_train, y_valid = train_test_split(X,y,test_size=0.2,random_state=25 )<compute_train_metric>
datagen = ImageDataGenerator(rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1)
 datagen.fit(X_train)
 train_batches = datagen.flow(X_train, y_train, batch_size = batch_size )
lr = LinearRegression()
lr.fit(X_train, y_train)
 pred = lr.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Linear Regression R2 Score: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_pred = cross_val_predict(lr,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)R2 Score: ', cv_accuracy)
<compute_train_metric>
model3 = first_cnn_model_keras(Adam(learning_rate = 0.001, amsgrad = True))
h3 = model3.fit_generator(train_batches, epochs = 40, verbose = 1,
validation_data =(X_valid, y_valid), callbacks = [callback_es])
 final_loss_first_adam_aug, final_acc_first_adam_aug = model3.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_first_adam_aug, final_acc_first_adam_aug))
ls = Lasso()
ls.fit(X_train, y_train)
 pred = ls.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Lasso Regression R2 Score: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Squared Error', np.sqrt(mse))
cv_pred = cross_val_predict(ls,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)Lasso Regression Accuracy: ', cv_accuracy )<compute_train_metric>
model4 = first_cnn_model_keras(RMSprop(lr=0.001))
h4 = model4.fit_generator(train_batches, epochs = 40, verbose = 1,
validation_data =(X_valid, y_valid), callbacks = [callback_es])
 final_loss_first_rmsprop_aug, final_acc_first_rmsprop_aug = model4.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_first_rmsprop_aug, final_acc_first_rmsprop_aug))
dt = DecisionTreeRegressor()
dt.fit(X_train, y_train)
 pred = dt.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Decision Tree R2 Score: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_pred = cross_val_predict(dt,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)Decision Tree Accuracy: ', cv_accuracy )<compute_train_metric>
def second_cnn_model_keras(optimizer):
model = Sequential()
model.add(Convolution2D(64, kernel_size =(5, 5), input_shape = input_shape, kernel_initializer = 'he_uniform'))
model.add(Activation('relu'))
model.add(Convolution2D(64, kernel_size =(5, 5), kernel_initializer = 'he_uniform'))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2), padding = 'same'))
model.add(Dropout(0.25))
model.add(Convolution2D(128, kernel_size =(3, 3), kernel_initializer = 'he_uniform'))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Convolution2D(128, kernel_size =(3, 3), kernel_initializer = 'he_uniform'))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2), padding = 'same'))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss = 'categorical_crossentropy', optimizer = optimizer, metrics=['accuracy'])
 return model
rf = RandomForestRegressor()
rf.fit(X_train, y_train)
 pred = rf.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Random Forest Regressor R2: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_pred = cross_val_predict(rf,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)Random Forest R2: ', cv_accuracy )<define_search_space>
model5 = second_cnn_model_keras(Adam(learning_rate = 0.001, amsgrad = True))
h5 = model5.fit(X_train, y_train, batch_size = batch_size, epochs = 20, verbose = 1,
validation_data =(X_valid, y_valid), callbacks = [callback_es])
 final_loss_second_adam, final_acc_second_adam = model5.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_second_adam, final_acc_second_adam))
rfr = RandomForestRegressor()
n_estimators = [int(x)for x in np.linspace(start = 50 , stop = 300, num = 5)]
max_features = [10,20,40,60,80,100,120]
max_depth = [int(x)for x in np.linspace(5, 10, num = 2)]
max_depth.append(None)
 bootstrap = [True, False]
r_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'bootstrap': bootstrap}
cv_random = RandomizedSearchCV(estimator=rfr, param_distributions=r_grid, n_iter = 20,
scoring='neg_mean_squared_error', cv = 3, verbose=2, random_state=42,
n_jobs=-1, return_train_score=True)
 cv_random.fit(X_train, y_train);
print(cv_random.best_params_)
 pred = cv_random.predict(X_valid)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_accuracy = r2_score(y_valid,pred)
 print('Random Forest Predict R2: ', cv_accuracy )<import_modules>
model6 = second_cnn_model_keras(RMSprop(lr=0.001))
h6 = model6.fit(X_train, y_train, batch_size = batch_size, epochs = 20, verbose = 1,
validation_data =(X_valid, y_valid), callbacks = [callback_es])
 final_loss_second_rmsprop, final_acc_second_rmsprop = model6.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_second_rmsprop, final_acc_second_rmsprop))
import h2o
from h2o.estimators.gbm import H2OGradientBoostingEstimator
from h2o.automl import H2OAutoML<set_options>
model7 = second_cnn_model_keras(Adam(learning_rate = 0.001, amsgrad = True))
h7 = model7.fit_generator(train_batches, epochs = 20, verbose = 1,
validation_data =(X_valid, y_valid),callbacks = [callback_es])
 final_loss_second_adam_aug, final_acc_second_adam_aug = model7.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_second_adam_aug, final_acc_second_adam_aug))
h2o.init()<prepare_output>
model8 = second_cnn_model_keras(RMSprop(lr=0.001))
h8 = model8.fit_generator(train_batches, epochs = 20, verbose = 1,
validation_data =(X_valid, y_valid), callbacks = [callback_es])
 final_loss_second_rmsprop_aug, final_acc_second_rmsprop_aug = model8.evaluate(X_valid, y_valid, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss_second_rmsprop_aug, final_acc_second_rmsprop_aug))
h2o_df=h2o.H2OFrame(train)
 h2o_df.head()<split>
models = ['first_cnn_adam', 'first_cnn_rmsprop', 'first_cnn_adam_aug', 'first_cnn_rmsprop_aug',
'second_cnn_adam', 'second_cnn_rmsprop', 'second_cnn_adam_aug', 'second_cnn_rmsprop_aug']
dict_values = {'loss': [final_loss_first_adam, final_loss_first_rmsprop, final_loss_first_adam_aug,
final_loss_first_rmsprop_aug, final_loss_second_adam, final_loss_second_rmsprop,
final_loss_second_adam_aug, final_loss_second_rmsprop_aug],
'accuracy': [final_acc_first_adam, final_acc_first_rmsprop, final_acc_first_adam_aug,
final_acc_first_rmsprop_aug, final_acc_second_adam, final_acc_second_rmsprop,
final_acc_second_adam_aug, final_acc_second_rmsprop_aug]}
df = pd.DataFrame(dict_values, index = models, columns = ['loss', 'accuracy'])
 df
splits = h2o_df.split_frame(ratios=[0.8],seed=1)
 h2o_train = splits[0]
h2o_valid = splits[1]<prepare_x_and_y>
model = [0]*10
for i in range(10):
model[i] = Sequential()
model[i].add(Convolution2D(64, kernel_size =(3, 3), input_shape = input_shape, kernel_initializer = 'he_uniform'))
model[i].add(Activation('relu'))
model[i].add(BatchNormalization())
 model[i].add(Convolution2D(64, kernel_size =(3, 3), kernel_initializer = 'he_uniform'))
model[i].add(Activation('relu'))
model[i].add(BatchNormalization())
 model[i].add(Convolution2D(64, kernel_size =(5, 5), kernel_initializer = 'he_uniform'))
model[i].add(Activation('relu'))
model[i].add(BatchNormalization())
 model[i].add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2)))
 model[i].add(Dropout(0.45))
model[i].add(Convolution2D(128, kernel_size =(3, 3), kernel_initializer = 'he_uniform'))
model[i].add(Activation('relu'))
model[i].add(BatchNormalization())
 model[i].add(Convolution2D(128, kernel_size =(3, 3), kernel_initializer = 'he_uniform'))
model[i].add(Activation('relu'))
model[i].add(BatchNormalization())
 model[i].add(Convolution2D(128, kernel_size =(5, 5), kernel_initializer = 'he_uniform'))
model[i].add(Activation('relu'))
model[i].add(BatchNormalization())
 model[i].add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2)))
 model[i].add(Dropout(0.45))
model[i].add(Flatten())
 model[i].add(Dense(512))
model[i].add(Activation('relu'))
model[i].add(Dropout(0.45))
model[i].add(Dense(1024))
model[i].add(Activation('relu'))
model[i].add(Dropout(0.45))
model[i].add(Dense(10))
model[i].add(Activation('softmax'))
model[i].compile(loss = 'categorical_crossentropy', optimizer = Adam(lr = 0.0005, amsgrad = True), metrics=['accuracy'] )
y = "revenue"
x = h2o_df.columns
x.remove(y )<choose_model_class>
callback_lrs = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 epochs = 40
history = [0]*10
for j in range(10):
X_train1, X_valid1, y_train1, y_valid1 = train_test_split(X_train, y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train1, y_train1, batch_size = batch_size), epochs = epochs, verbose = 0,
validation_data =(X_valid1, y_valid1), callbacks = [callback_lrs])
 print('CNN:', j+1, 'Epochs =', epochs, 'Train accuracy:', max(history[j].history['accuracy']), 'Validation accuracy:', max(history[j].history['val_accuracy']))
<train_model><EOS> 
results = np.zeros(( X_test.shape[0],10))
for j in range(10):
results = results + model[j].predict(X_test)
 y_test_class = np.argmax(results, axis = 1)
 submission = pd.DataFrame({'ImageId': list(range(1, len(y_test_class)+1)) , 'Label': np.array(y_test_class)})
 submission.to_csv('submission.csv', index=False)
 print(submission )