Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<save_to_csv>
import numpy
from keras import backend as K
from keras.datasets import mnist
from keras.utils import np_utils
def create_sub() :
results = model.predict(testx)
 results = np.argmax(results,axis = 1)
 df_sub = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 df_sub['label'] = results
df_sub.to_csv("submission.csv",index=False )<prepare_output>
from keras.layers import Dense, Dropout,Flatten
from keras.layers.convolutional import Conv2D, MaxPooling2D
create_sub()<set_options>
from keras.models import Sequential
import pandas as pd
%matplotlib inline
DATA_DIR = '.. /input/recruit-restaurant-visitor-forecasting'
def rmsle(y_test, y_pred):
return np.sqrt(mean_squared_error(y_test, y_pred))
calendar.setfirstweekday(6)
 def get_week_of_month(date_str):
year = int(date_str[:4])
 month = int(date_str[5:7])
 day = int(date_str[8:10])
 x = np.array(calendar.monthcalendar(year, month))
week_of_month = np.where(x==day)[0][0] + 1
return(week_of_month)
 def save_submission(model, train, y, test, model_name):
model.fit(train, y)
 pred = model.predict(test)
 pred = np.expm1(pred)
 data_dir = DATA_DIR
submission_path = os.path.join(data_dir, "sample_submission.csv")
 submission = pd.read_csv(submission_path)
 submission['visitors'] = pred
print(submission.head())
 submission_csv_path = '{}/submission_{}_RMSLE.csv'.format(data_dir, model_name)
 submission.to_csv(submission_csv_path, index=False )<load_from_csv>
K.set_image_data_format('channels_last')
 numpy.random.seed(0 )
air_visit = pd.read_csv(os.path.join(DATA_DIR, 'air_visit_data.csv.zip'))
air_reserve = pd.read_csv(os.path.join(DATA_DIR, 'air_reserve.csv.zip'))
hpg_reserve = pd.read_csv(os.path.join(DATA_DIR, 'hpg_reserve.csv.zip'))
air_store = pd.read_csv(os.path.join(DATA_DIR, 'air_store_info.csv.zip'))
hpg_store = pd.read_csv(os.path.join(DATA_DIR, 'hpg_store_info.csv.zip'))
store_id_map = pd.read_csv(os.path.join(DATA_DIR, 'store_id_relation.csv.zip'))
date_info = pd.read_csv(os.path.join(DATA_DIR, 'date_info.csv.zip'))
submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv.zip'))<feature_engineering>
X = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 y = X["label"]
X.drop(["label"], inplace = True, axis = 1 )
air_visit_count = len(air_visit)
 train_data = air_visit.copy()
test_data = submission.copy()
test_data["air_store_id"] = test_data.apply(lambda x: '_'.join(x['id'].split('_')[:2]), axis=1)
 test_data["visit_date"] = test_data.apply(lambda x:x['id'].split("_")[-1], axis=1)
 test_data.drop("id",axis=1, inplace=True)
 concat_data = pd.concat([train_data, test_data])
 concat_data<merge>
X_train, X_test, y_train, y_test = train_test_split(X.values, y.values, test_size=0.2 , random_state=42 )
concat_data = pd.merge(concat_data, date_info, how="left", left_on="visit_date", right_on="calendar_date")
 concat_data.drop("calendar_date", axis=1, inplace=True)
 concat_data.head()<merge>
X_train = X_train.reshape(X_train.shape[0], 28, 28 , 1 ).astype('float32')
 X_test = X_test.reshape(X_test.shape[0], 28, 28 , 1 ).astype('float32' )
concat_data = pd.merge(concat_data, air_store, how="left", on="air_store_id")
 concat_data = pd.merge(concat_data, store_id_map, how="left", on="air_store_id" )<merge>
model = Sequential()
model.add(Conv2D(100, kernel_size=3, padding="valid", input_shape=(28, 28, 1), activation = 'relu'))
concat_data = pd.merge(concat_data, hpg_store, how="left", on="hpg_store_id")
 concat_data<create_dataframe>
model.add(Conv2D(100, kernel_size=3, padding="valid", activation = 'relu'))
model.add(Conv2D(100, kernel_size=3, padding="valid", activation = 'relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(1, 1)) )
air_reserve_data = air_reserve.copy()
air_reserve_data["visit_datetime"] = air_reserve_data["visit_datetime"].str[:10]
air_reserve_data = air_reserve_data[["air_store_id", "visit_datetime", "reserve_visitors"]] \ 
.groupby(by=["air_store_id", 'visit_datetime'])\ 
.sum() \ 
.reset_index()
air_reserve_data = air_reserve_data.rename(columns = {'reserve_visitors': 'air_reserve_visitors'})
 air_reserve_data = air_reserve_data.rename(columns = {'visit_datetime': 'visit_date'})
 pd.merge(concat_data, air_reserve_data, how="left", on=['air_store_id', 'visit_date'])
 concat_data = pd.merge(concat_data, air_reserve_data, how="left", on=['air_store_id', 'visit_date'] )<create_dataframe>
model.add(Flatten())
 model.add(Dense(units= 500, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10))
model.add(Activation("softmax"))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] )
hpg_reserve_data = hpg_reserve.copy()
hpg_reserve_data["visit_datetime"] = hpg_reserve_data["visit_datetime"].str[:10]
hpg_reserve_data = hpg_reserve_data[["hpg_store_id", "visit_datetime", "reserve_visitors"]] \ 
.groupby(by=["hpg_store_id", 'visit_datetime'])\ 
.sum() \ 
.reset_index()
hpg_reserve_data = hpg_reserve_data.rename(columns = {'reserve_visitors': 'hpg_reserve_visitors'})
 hpg_reserve_data = hpg_reserve_data.rename(columns = {'visit_datetime': 'visit_date'})
 hpg_reserve_data["visit_date"] = hpg_reserve_data["visit_date"].str[:10]
concat_data = pd.merge(concat_data, hpg_reserve_data, how="left", on=['hpg_store_id', 'visit_date'] )<count_missing_values>
y_train = np_utils.to_categorical(y_train ).astype('int32')
 y_test = np_utils.to_categorical(y_test )
concat_data.isna().sum()<drop_column>
callbacks = [
keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-3,
patience=25,
verbose=1)
 ]
concat_data = concat_data.drop(["hpg_area_name", "hpg_genre_name", "latitude_y", "longitude_y"], axis=1 )<data_type_conversions>
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 X_train = X_train.reshape(-1,28,28,1)
 X_test = X_test.reshape(-1,28,28,1)
 datagen = ImageDataGenerator(
 featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=10,
fill_mode='nearest',
validation_split = 0.2 
)
 datagen.fit(X_train)
 train_generator = datagen.flow(X_train, y_train, batch_size=60, subset='training')
 validation_generator = datagen.flow(X_train, y_train, batch_size=60, subset='validation')
 history = model.fit_generator(generator=train_generator,
validation_data=validation_generator,
use_multiprocessing=True,
steps_per_epoch = len(train_generator)/ 60,
validation_steps = len(validation_generator)/ 60,
epochs = 300,
workers=-1 )
concat_data[["air_reserve_visitors"]] = concat_data[["air_reserve_visitors"]].fillna(0)
 concat_data[["hpg_reserve_visitors"]] = concat_data[["hpg_reserve_visitors"]].fillna(0)
 concat_data.isna().sum()<feature_engineering>
scores = model.evaluate(X_test, y_test, verbose = 10)
 print(scores)
<feature_engineering><EOS> 
test_set =(test.values ).reshape(-1, 28, 28 , 1 ).astype('float32')
 res = model.predict(test_set)
 res = numpy.argmax(res,axis = 1)
 res = pd.Series(res, name="Label")
 submission = pd.concat([pd.Series(range(1 ,28001),name = "ImageId"), res],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False)
 submission.head(10 )