Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
print(os.listdir(".. /input"))
sub = pd.read_csv(dfk.iloc[1, 2])
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp' )<save_to_csv>
def preprocess(data):
for im in tqdm(range(data.shape[0])) :
im_array=data[im].reshape(28,28)
 im_array=exposure.adjust_sigmoid(exposure.adjust_sigmoid(exposure.adjust_sigmoid(exposure.adjust_sigmoid(im_array))))
data[im]=im_array.reshape(28,28,1)
 return data
simple_accurate_99 = pd.read_csv(dfk.iloc[5, 2])
 sub['floor'] = simple_accurate_99['floor'].values
sub.to_csv('b9530cm99.csv', index=False)
 b9530cm99 = sub<feature_engineering>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 testIndex=test.index
ylabel_train=train['label']
train=train.drop('label',axis=1 ).values.reshape(train.shape[0],28,28,1)
 test=test.values.reshape(test.shape[0],28,28,1)
 print('Adjusting training and test images')
 train=train/255.0
test=test/255.0
sub = pd.read_csv(dfk.iloc[2, 2])
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp' )<save_to_csv>
X_train,X_test,y_train,y_test=train_test_split(train,to_categorical(ylabel_train),test_size=0.2,random_state=42)
 print(X_train.shape,X_test.shape )
simple_accurate_99 = pd.read_csv(dfk.iloc[5, 2])
 sub['floor'] = simple_accurate_99['floor'].values
sub.to_csv('c8500cm99.csv', index=False)
 c8500cm99 = sub<feature_engineering>
trainDatagen=ImageDataGenerator(rotation_range=15,width_shift_range=0.1,height_shift_range=0.1,zoom_range=0.2,fill_mode='nearest')
 valDatagen=ImageDataGenerator(rotation_range=15,width_shift_range=0.1,height_shift_range=0.1,zoom_range=0.2,fill_mode='nearest' )
sub = pd.read_csv(dfk.iloc[3, 2])
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp' )<save_to_csv>
model=Sequential()
model.add(Conv2D(64,3,activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(128,3,activation='relu'))
model.add(MaxPooling2D(( 3,3),strides=(3,3)))
 model.add(Dropout(0.25))
model.add(Conv2D(256,3,activation='relu'))
model.add(Conv2D(512,1,activation='relu'))
model.add(MaxPooling2D(( 3,3),strides=(3,3)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(100,activation='relu'))
model.add(Dense(100,activation='relu'))
model.add(Dense(10,activation='softmax'))
simple_accurate_99 = pd.read_csv(dfk.iloc[5, 2])
 sub['floor'] = simple_accurate_99['floor'].values
sub.to_csv('d8418cm99.csv', index=False)
 d8418cm99 = sub<feature_engineering>
model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'] )
sub = pd.read_csv(dfk.iloc[4, 2])
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp' )<save_to_csv>
estop=EarlyStopping(monitor='val_acc',patience=10,verbose=1)
 lrreduce=ReduceLROnPlateau(monitor='val_acc',verbose=1,factor=0.5,min_lr=1e-10,patience=3 )
simple_accurate_99 = pd.read_csv(dfk.iloc[5, 2])
 sub['floor'] = simple_accurate_99['floor'].values
sub.to_csv('e8333cm99.csv', index=False)
 e8333cm99 = sub<feature_engineering>
bSize=128
model.fit_generator(trainDatagen.flow(x=X_train,y=y_train,batch_size=bSize),steps_per_epoch=X_train.shape[0]*20/bSize,epochs=500,validation_data=(X_test,y_test),callbacks=[estop,lrreduce] )
sub = pd.read_csv(dfk.iloc[5, 2])
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp' )<save_to_csv>
predictions=model.predict(test)
 predictions_df=pd.DataFrame({'ImageId':testIndex+1,'Label':np.argmax(predictions,axis=1)})
 predictions_df[['ImageId','Label']].to_csv('Submission_CNN_LargeAugmentation.csv',index=False )
sub.to_csv('f8073cm99.csv', index=False)
 f8073cm99 = sub<feature_engineering>
model.save('MNIST_CNN_LargeAugmentation.kps' )