Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
sns.set(style='white', context='notebook', palette='deep')
 plt.rcParams['image.cmap']='gray'
sns.set_style("whitegrid")
 warnings.filterwarnings('ignore')
<load_from_csv>
col_names = ['label']+[str(x)for x in range(784)]
df = pd.concat([
pd.read_csv('.. /input/mnist-in-csv/mnist_train.csv', names=col_names, header=0),
pd.read_csv('.. /input/mnist-in-csv/mnist_test.csv', names=col_names, header=0),
pd.read_csv('.. /input/digit-recognizer/train.csv', names=col_names, header=0),
pd.read_csv('.. /input/digit-recognizer/test.csv', names=col_names[1:], header=0),
], axis=0, sort=False, ignore_index=True )
train_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/train.csv.zip',parse_dates=True)
 sample_submission=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/sampleSubmission.csv.zip')
 features_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/features.csv.zip',parse_dates=True)
 stores_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/stores.csv')
 test_data=pd.read_csv('/kaggle/input/walmart-recruiting-store-sales-forecasting/test.csv.zip' )<data_type_conversions>
num_classes = 10
X, y = df[col_names[1:]], df[col_names[0]]
df = pd.DataFrame()
for i in tq.tqdm(range(1,46)) :
model=Prophet()
filled=features_data[(( features_data['Store']==i)&(features_data['Date']<'2013-05-03')) ][['Date','CPI']]
tserie = filled.rename(columns = {'Date': 'ds', 'CPI': 'y'}, inplace = False)
 tserie =tserie.sort_values(by=['ds'])
 tserie['ds'] = pd.to_datetime(tserie['ds'])
 model.fit(tserie)
 future_dates=model.make_future_dataframe(periods=13,freq = 'W',include_history =True)
 future_dates['ds'] = future_dates['ds'].apply(lambda x: x + timedelta(days=5))
prediction=model.predict(future_dates)
 df=df.append(prediction)
 df.reset_index(drop=True)
 features_data['CPI1']=np.nan
for i,j in enumerate(df['yhat']):
features_data['CPI1'].iloc[i]=j
features_data=features_data.drop(['CPI'],axis=1)
 features_data = features_data.rename(columns = {'CPI1': 'CPI'} )<data_type_conversions>
train = y.notna()
test = ~train
y_matrix =(y[:,None] == range(num_classes)).astype(int)
 Xtrain, ytrain = X[train], y_matrix[train]
Xtest , ytest = X[test] , y_matrix[test]
df = pd.DataFrame()
for i in tq.tqdm(range(1,46)) :
model=Prophet()
filled=features_data[(( features_data['Store']==i)&(features_data['Date']<'2013-05-03')) ][['Date','Unemployment']]
tserie = filled.rename(columns = {'Date': 'ds', 'Unemployment': 'y'}, inplace = False)
 tserie =tserie.sort_values(by=['ds'])
 tserie['ds'] = pd.to_datetime(tserie['ds'])
 model.fit(tserie)
 future_dates=model.make_future_dataframe(periods=13,freq = 'W',include_history =True)
 future_dates['ds'] = future_dates['ds'].apply(lambda x: x + timedelta(days=5))
prediction=model.predict(future_dates)
 df=df.append(prediction)
 df.reset_index(drop=True)
 features_data['Unemployment1']=np.nan
for i,j in enumerate(df['yhat']):
features_data['Unemployment1'].iloc[i]=j
features_data=features_data.drop(['Unemployment'],axis=1)
 features_data = features_data.rename(columns = {'Unemployment1': 'Unemployment'} )<merge>
def baseline_model() :
model = Sequential()
model.add(Conv2D(32, kernel_size=(6, 6), strides=(2, 2), activation='relu',input_shape=Xtrain[0].shape))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Conv2D(64,(5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(100, activation='relu'))
model.add(Dense(25, activation='relu'))
model.add(Dense(num_classes, activation='sigmoid'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
stores = stores_data.merge(features_data, on ='Store' , how = 'left')
 final_data_train = train_data.merge(stores, on = ['Store', 'Date', 'IsHoliday'], how = 'left' )<merge>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
brightness_range=None,
shear_range=5,
zoom_range=-0.4,
fill_mode='nearest',
horizontal_flip=False,
vertical_flip=False,
rescale=None,
preprocessing_function=None,
data_format=None,
validation_split=0.2
)
stores = stores_data.merge(features_data, on ='Store' , how = 'left')
 final_data_test = test_data.merge(stores, on = ['Store', 'Date', 'IsHoliday'], how = 'left' )<categorify>
datagen.fit(Xtrain )
def markdown_imputation(final_data):
final_data.loc[final_data.MarkDown1.isnull() ,'MarkDown1']= 0
final_data.loc[final_data.MarkDown2.isnull() ,'MarkDown2']= 0
final_data.loc[final_data.MarkDown3.isnull() ,'MarkDown3']= 0
final_data.loc[final_data.MarkDown4.isnull() ,'MarkDown4']= 0
final_data.loc[final_data.MarkDown5.isnull() ,'MarkDown5']= 0
return final_data<split>
augmentation = True
if augmentation:
history = estimator.fit_generator(
 datagen.flow(Xtrain, ytrain, batch_size=10),
steps_per_epoch=Xtrain.shape[0],
epochs=10 
)
 else:
history = estimator.fit(
 Xtrain, ytrain,
batch_size=32, epochs=10,
validation_split=len(ytest)/len(y)
 )
def train_temp_bins(final_data):
temp_100_110_f=final_data[(( final_data.Temperature>100)&(final_data.Temperature< 110)) ].Weekly_Sales.sum()
temp_90_100_f=final_data[(( final_data.Temperature>90)&(final_data.Temperature< 100)) ].Weekly_Sales.sum()
temp_80_90_f=final_data[(( final_data.Temperature>80)&(final_data.Temperature< 90)) ].Weekly_Sales.sum()
temp_70_80_f=final_data[(( final_data.Temperature>70)&(final_data.Temperature< 80)) ].Weekly_Sales.sum()
temp_60_70_f=final_data[(( final_data.Temperature>60)&(final_data.Temperature< 70)) ].Weekly_Sales.sum()
temp_50_60_f=final_data[(( final_data.Temperature>50)&(final_data.Temperature< 60)) ].Weekly_Sales.sum()
temp_40_50_f=final_data[(( final_data.Temperature>40)&(final_data.Temperature< 50)) ].Weekly_Sales.sum()
temp_30_40_f=final_data[(( final_data.Temperature>30)&(final_data.Temperature< 40)) ].Weekly_Sales.sum()
temp_0_30_f=final_data[(( final_data.Temperature>0)&(final_data.Temperature< 30)) ].Weekly_Sales.sum()
temp_less_than_0_f=final_data[(( final_data.Temperature>-10)&(final_data.Temperature< 0)) ].Weekly_Sales.sum()
final_data['Temp_bins'] = np.nan
final_data.loc[(( final_data.Temperature>-10)&(final_data.Temperature<0)) ,'Temp_bins']= temp_less_than_0_f
final_data.loc[(( final_data.Temperature>0)&(final_data.Temperature< 30)) ,'Temp_bins']= temp_0_30_f
final_data.loc[(( final_data.Temperature>30)&(final_data.Temperature< 40)) ,'Temp_bins']= temp_30_40_f
final_data.loc[(( final_data.Temperature>40)&(final_data.Temperature< 50)) ,'Temp_bins']= temp_40_50_f
final_data.loc[(( final_data.Temperature>50)&(final_data.Temperature< 60)) ,'Temp_bins']= temp_50_60_f
final_data.loc[(( final_data.Temperature>60)&(final_data.Temperature< 70)) ,'Temp_bins']= temp_60_70_f
final_data.loc[(( final_data.Temperature>70)&(final_data.Temperature< 80)) ,'Temp_bins']= temp_70_80_f
final_data.loc[(( final_data.Temperature>80)&(final_data.Temperature< 90)) ,'Temp_bins']= temp_80_90_f
final_data.loc[(( final_data.Temperature>90)&(final_data.Temperature< 100)) ,'Temp_bins']= temp_90_100_f
final_data.loc[(( final_data.Temperature>100)&(final_data.Temperature< 110)) ,'Temp_bins']= temp_100_110_f
final_data.loc[final_data.Temp_bins.isnull() ,'Temp_bins']= 0
list1=[temp_less_than_0_f,temp_0_30_f,temp_30_40_f,temp_40_50_f,temp_50_60_f,temp_60_70_f,temp_70_80_f,temp_80_90_f,temp_90_100_f,temp_100_110_f]
return final_data,list1<feature_engineering>
ytest = estimator.predict_classes(Xtest )
def test_temp_bins(final_data,list1):
final_data['Temp_bins'] = np.nan
final_data.loc[(( final_data.Temperature>-10)&(final_data.Temperature<0)) ,'Temp_bins']= list1[0]
final_data.loc[(( final_data.Temperature>0)&(final_data.Temperature< 30)) ,'Temp_bins']= list1[1]
final_data.loc[(( final_data.Temperature>30)&(final_data.Temperature< 40)) ,'Temp_bins']= list1[2]
final_data.loc[(( final_data.Temperature>40)&(final_data.Temperature< 50)) ,'Temp_bins']= list1[3]
final_data.loc[(( final_data.Temperature>50)&(final_data.Temperature< 60)) ,'Temp_bins']= list1[4]
final_data.loc[(( final_data.Temperature>60)&(final_data.Temperature< 70)) ,'Temp_bins']= list1[5]
final_data.loc[(( final_data.Temperature>70)&(final_data.Temperature< 80)) ,'Temp_bins']= list1[6]
final_data.loc[(( final_data.Temperature>80)&(final_data.Temperature< 90)) ,'Temp_bins']= list1[7]
final_data.loc[(( final_data.Temperature>90)&(final_data.Temperature< 100)) ,'Temp_bins']= list1[8]
final_data.loc[(( final_data.Temperature>100)&(final_data.Temperature< 110)) ,'Temp_bins']= list1[9]
final_data.loc[final_data.Temp_bins.isnull() ,'Temp_bins']= 0
return final_data<feature_engineering>
submit = pd.DataFrame(data={'ImageId': range(1, ytest.shape[0]+1), 'Label': ytest} )
def split(final_data):
final_data['Date'] = pd.to_datetime(final_data['Date'])
 final_data['Year'] = final_data['Date'].dt.year
final_data['Month']= final_data['Date'].dt.month
final_data['Week'] = final_data['Date'].dt.week
final_data['Day'] = final_data['Date'].dt.day
return final_data<feature_engineering>
submit.to_csv("submit.csv", index=None )