Kaggle competition: Rossmann Store Sales
<SOS> metric: RootMeanSquarePercentageError Kaggle data source: rossmann-store-sales<set_options>
%matplotlib inline
%%time
agents = [ RandomSeedSearch(verbose=False), random_agent_unseeded ]
results = evaluate(
 "rps",
agents,
configuration={
"episodeSteps": 1000,
"tieRewardThreshold": 1,
"actTimeout": 1000,
},
num_episodes=30 
)
 results = np.array(results ).reshape(( -1,2))
results[ results == None ] = -1
totals = np.sum(results, axis=0)
 std = np.std(results, axis=0 ).round(2)
 winrate = [ np.sum(results[:,0]-20 > results[:,1]),
np.sum(results[:,0]+20 < results[:,1])]
print([ getattr(agent, '__name__', agent.__class__.__name__)for agent in agents ])
 print('winrate', winrate, '/', len(results))
print('totals ', totals)
 print('std ', std )<load_from_zip>
df_train = pd.read_csv(".. /input/train.csv")
 df_store = pd.read_csv(".. /input/store.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df_train.head()
%%time
%%bash
GZIP=-9 tar cfz main.py.tar.gz main.py *.npy
ls -lah *.tar.gz *.npy *.py
tar -tvf *.tar.gz<import_modules>
closed_store_ids = df_test["Id"][df_test["Open"] == 0].values
closed_store_ids
import numpy as np
import pandas as pd
import os<load_from_csv>
df_train['Year'] = df_train['Date'].apply(lambda x: int(x[:4]))
df_train['Month'] = df_train['Date'].apply(lambda x: int(x[5:7]))
df_train['Day'] = df_train['Date'].apply(lambda x: int(x[8:]))
train = pd.read_csv(".. /input/invasive-species-monitoring/train_labels.csv")
 train["name"] = train["name"].apply(lambda x : ".. /input/invasive-species-monitoring/train/" + str(x)+ ".jpg")
 train["invasive"] = train["invasive"].astype("str" )<prepare_x_and_y>
df_train["HolidayBin"] = df_train['StateHoliday'].map({"0": 0, "a": 1, "b": 1, "c": 1} )
train_image_gen = ImageDataGenerator(rescale = 1/255, validation_split=0.022, rotation_range=270, horizontal_flip=True)
 batch_size = 6
image_size = 800
train_generator = train_image_gen.flow_from_dataframe(dataframe=train, batch_size=batch_size, target_size=(image_size,image_size), x_col="name", y_col="invasive", subset="training", class_mode="binary")
 val_generator = train_image_gen.flow_from_dataframe(dataframe=train, batch_size=batch_size, target_size=(image_size,image_size), x_col="name", y_col="invasive", subset="validation", class_mode="binary" )<choose_model_class>
total_sales_customers = df_train.groupby('Store')['Sales', 'Customers'].sum()
total_sales_customers.head()
model = Sequential()
model.add(InceptionResNetV2(weights="imagenet", include_top=False, pooling="avg", input_shape=(image_size, image_size, 3)))
 model.add(Dropout(0.5))
model.add(Dense(1, activation="sigmoid"))<choose_model_class>
df_total_sales_customers = pd.DataFrame({'Sales': total_sales_customers['Sales'],
'Customers': total_sales_customers['Customers']},
index = total_sales_customers.index)
 df_total_sales_customers = df_total_sales_customers.reset_index()
df_total_sales_customers.head()
model.compile(optimizer=Adam(lr=0.0003, decay=2e-7), loss='binary_crossentropy',metrics=['accuracy'] )<train_model>
avg_sales_customers = df_train.groupby('Store')['Sales', 'Customers'].mean()
avg_sales_customers.head()
modelcheck = ModelCheckpoint("best.h5", save_best_only=True)
 model.fit_generator(train_generator, steps_per_epoch=int(np.ceil(train_generator.n/batch_size)) , validation_data = val_generator, validation_steps = int(np.ceil(val_generator.n/batch_size)) , epochs=6, callbacks=[modelcheck] )<save_to_csv>
df_avg_sales_customers = pd.DataFrame({'Sales': avg_sales_customers['Sales'],
'Customers': avg_sales_customers['Customers']},
index = avg_sales_customers.index)
 df_avg_sales_customers = df_avg_sales_customers.reset_index()
df_stores_avg = df_avg_sales_customers.join(df_store.set_index('Store'), on='Store')
 df_stores_avg.head()
sub = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 test = pd.DataFrame({"name": sub["name"].apply(lambda x : ".. /input/invasive-species-monitoring/test/" + str(x)+ ".jpg")})
 test_gen = ImageDataGenerator(rescale = 1./255)
 test_generator = test_gen.flow_from_dataframe(dataframe = test, shuffle=False, x_col = "name", y_col = None, class_mode=None, batch_size = batch_size, target_size =(image_size, image_size))
model.load_weights("best.h5")
 preds = model.predict_generator(test_generator, steps=int(np.ceil(test_generator.n/batch_size)) , workers = 2)
 sub["invasive"] = preds
sub.to_csv("submission.csv", index=False )<set_options>
df_stores_new = df_total_sales_customers.join(df_store.set_index('Store'), on='Store')
 df_stores_new.head()
%matplotlib inline
master = pd.read_csv(".. /input/invasive-species-monitoring/train_labels.csv")
 master.head()<define_variables>
df_test['Year'] = df_test['Date'].apply(lambda x: int(x[:4]))
df_test['Month'] = df_test['Date'].apply(lambda x: int(x[5:7]))
df_test['Day'] = df_test['Date'].apply(lambda x: int(x[8:]))
df_test["HolidayBin"] = df_test.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1})
 del df_test['Date']
del df_test['StateHoliday']
df_test.head()
img_path = ".. /input/invasive-species-monitoring/train/"
y = []
file_paths = []
for i in range(len(master)) :
file_paths.append(img_path + str(master.iloc[i][0])+'.jpg')
 y.append(master.iloc[i][1])
 y = np.array(y )<load_from_csv>
del df_train['Date']
del df_train['StateHoliday']
sample_submission = pd.read_csv(".. /input/invasive-species-monitoring/sample_submission.csv")
 img_path = ".. /input/invasive-species-monitoring/test/"
test_names = []
file_paths = []
for i in range(len(sample_submission)) :
test_names.append(sample_submission.iloc[i][0])
 file_paths.append(img_path + str(int(sample_submission.iloc[i][0])) +'.jpg')
 test_names = np.array(test_names )<randomize_order>
a = list()
for i in df_test['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_test['CompetitionDistance'] = a
data_num = len(y)
 random_index = np.random.permutation(data_num)
 x_shuffle = []
y_shuffle = []
for i in range(data_num):
x_shuffle.append(x[random_index[i]])
 y_shuffle.append(y[random_index[i]])
 x = np.array(x_shuffle)
 y = np.array(y_shuffle )<data_type_conversions>
a = list()
for i in df_train['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_train['CompetitionDistance'] = a
df_train['CompetitionDistance'] = df_train['CompetitionDistance'].fillna(df_train['CompetitionDistance'].mean() )
x_train = x_train.astype('float32')
 x_test = x_test.astype('float32')
 x_train /= 255
x_test /= 255<choose_model_class>
df_train['CompetitionDistance'] = np.log(df_train['CompetitionDistance'])
 df_test['CompetitionDistance'] = np.log(df_test['CompetitionDistance'] )
img_rows, img_cols, img_channel = 224, 224, 3
base_model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(img_rows, img_cols, img_channel))<choose_model_class>
train_stores = dict(list(df_train.groupby('Store')))
 test_stores = dict(list(df_test.groupby('Store')) )
add_model = Sequential()
add_model.add(Flatten(input_shape=base_model.output_shape[1:]))
add_model.add(Dense(256, activation='relu'))
add_model.add(Dense(1, activation='sigmoid'))
model = Model(inputs=base_model.input, outputs=add_model(base_model.output))
model.compile(loss='binary_crossentropy', optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
 model.summary()<train_model>
best_list_max_depth = []
best_list_n_estimators = []
for i in test_stores:
store = train_stores[i]
X_train = store.drop(["Sales", "Store", "Customers"],axis=1)
 Y_train = store["Sales"]
X_test = test_stores[i].copy()
store_ids = X_test["Id"]
X_test.drop(["Id","Store"], axis=1,inplace=True)
 X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean())
 estimator = RandomForestRegressor(random_state=123, criterion = 'mse')
 params = {'n_estimators': range(5, 20), 'max_depth': range(5, 25)}
grid = GridSearchCV(estimator, params ).fit(X_train, Y_train)
 best_list_max_depth.append(grid.best_params_['max_depth'])
 best_list_n_estimators.append(grid.best_params_['n_estimators'])
 print("score", grid.best_score_)
 print("params", grid.best_params_ )
batch_size = 32
epochs = 50
train_datagen = ImageDataGenerator(
 rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True)
 train_datagen.fit(x_train)
 history = model.fit_generator(
 train_datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
)<train_model>
res_max_depth = round(np.array(best_list_max_depth ).mean())
 res_n_estimators = round(np.array(best_list_n_estimators ).mean() )
print("Training loss: {:.2f} / Validation loss: {:.2f}".\
format(history.history['loss'][-1], history.history['val_loss'][-1]))
print("Training accuracy: {:.2f}% / Validation accuracy: {:.2f}%".\
format(100*history.history['acc'][-1], 100*history.history['val_acc'][-1]))<data_type_conversions>
best_max_depth = round(np.array(best_list_max_depth ).mean())
 best_n_estimators = round(np.array(best_list_n_estimators ).mean())
 print(best_max_depth)
 print(best_n_estimators )
<predict_on_test><EOS> 
submission = pd.Series()
for i in test_stores:
store = train_stores[i]
X_train = store.drop(["Sales", "Store", "Customers"],axis=1)
 Y_train = store["Sales"]
X_test = test_stores[i].copy()
store_ids = X_test["Id"]
X_test.drop(["Id","Store"], axis=1,inplace=True)
 X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean())
 rfr = RandomForestRegressor(random_state=123, n_estimators = best_n_estimators,
max_depth = best_max_depth, criterion = 'mse')
 rfr.fit(X_train, Y_train)
 Y_pred = rfr.predict(X_test)
 submission = submission.append(pd.Series(Y_pred, index=store_ids))
submission = pd.DataFrame({ "Id": submission.index, "Sales": submission.values})
 submission.to_csv('rossmann_submission_good.csv', index=False)
 print(submission.shape )