Kaggle competition: Digit Recognizer
print(df.duplicated(keep=False ).value_counts())
 df = df.drop_duplicates()<data_type_conversions>
train = pd.read_csv('.. /input/train.csv')
 labels = train.iloc[:,0].values.astype('int32')
 X_train =(train.iloc[:,1:].values ).astype('float32')
 X_test =(pd.read_csv('.. /input/test.csv' ).values ).astype('float32')
 X_train = X_train.reshape(-1,28,28,1)
 X_test = X_test.reshape(-1,28,28,1)
 y_train = tf.keras.utils.to_categorical(labels)
 print("Check data")
 print(labels)
 print(X_train[0].shape)
 print(y_train )
def convert_dataframe(df):

 df["Dates"] = pd.to_datetime(df["Dates"],infer_datetime_format=True)
 df['Date'] = df['Dates'].dt.date
df["Year"] = df["Dates"].dt.year.astype(np.int32)
 df["Month"] = df["Dates"].dt.month.astype(np.int32)
 df["Day"] = df["Dates"].dt.day.astype(np.int32)
 df["Hour"] = df["Dates"].dt.hour.astype(np.int32)
 df["Minute"] = df["Dates"].dt.minute.astype(np.int32)
 df["Day_week_numeric"] = df["Dates"].dt.dayofweek.astype(np.int32)
 df["Weekend"]= np.where(( df["Day_week_numeric"] >= 4)&(df["Day_week_numeric"] <=6),1,0)
 df["count_days"] =(df['Date'] - df['Date'].min() ).apply(lambda x: x.days)
 df["Block"] = df.Address.str.contains("Block" ).astype(np.int32)
 df = df.drop(["Date","Address"],axis=1)
 return df<create_dataframe>
( train_imagesRaw, train_labelsRaw),(test_imagesRaw, test_labelsRaw)= mnist.load_data()
df_date = convert_dataframe(df )<categorify>
X_train_keras = train_imagesRaw.reshape(-1,28,28,1)
 X_test_keras = test_imagesRaw.reshape(-1,28,28,1)
 print("X_train_keras",X_train_keras.shape)
 print("X_test_keras",X_test_keras.shape)
 train_labels_keras = tf.keras.utils.to_categorical(train_labelsRaw)
 test_labels_keras = tf.keras.utils.to_categorical(test_labelsRaw)
 print("train_labels_keras ",train_labels_keras.shape)
 print("test_labels_keras ", test_labels_keras.shape )
label_cat = LabelEncoder()
df_date["Category_encode"] = label_cat.fit_transform(df_date.Category)
 label_dist = LabelEncoder()
df_date["PdDistric_encode"] = label_dist.fit_transform(df_date.PdDistrict)
<filter>
train_images = np.concatenate(( X_train_keras,X_train,X_test_keras), axis=0)
 print("new Concatenated train_images ", train_images.shape)
 print("_"*50)
 train_labels = np.concatenate(( train_labels_keras,y_train,test_labels_keras), axis=0)
 print("new Concatenated train_labels ", train_labels.shape )
df_outliers = df_date.loc[df_date.Y < 90.].copy()<train_model>
scale = np.max(train_images)
 train_images /= scale
X_test /= scale
print("Max: {}".format(scale))
X = df_outliers.drop(["Dates","Category","Descript","DayOfWeek","PdDistrict","Resolution","Category_encode"],axis=1 ).copy()
y = df_outliers["Category_encode"]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.25, random_state = 21)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_val = scaler.transform(X_val)
 kmeans = KMeans(n_clusters=6,random_state=0 ).fit(X_train)
<prepare_output>
input_size = train_images.shape
n_logits = train_labels.shape[1]
print("Input: {}".format(input_size))
print("Output: {}".format(n_logits))
X_train_df = pd.DataFrame(X_train)
 X_val_df = pd.DataFrame(X_val)
 X_train_df["Kmean"] = kmeans.labels_
X_val_df["Kmean"] = kmeans.predict(X_val )<train_model>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 30
batch_size = 512
X_train, X_val, Y_train, Yval = train_test_split(train_images, train_labels, train_size = 0.90)
 datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
classifier = RandomForestClassifier(n_jobs = -1,random_state =50,max_depth=10,max_features="auto",min_samples_split=4)
 classifier.fit(X_train_df, y_train )<predict_on_test>
model = tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3),padding='same', activation='relu', input_shape=input_size[1:]))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None))
model.add(tf.keras.layers.Conv2D(64, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(n_logits, activation='softmax'))
optimizer = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 model.summary()
predict_proba = classifier.predict_proba(X_val_df)
 log_loss(y_val,predict_proba )<load_from_csv>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Yval),
verbose = 1, steps_per_epoch=X_train.shape[0] )
<categorify><EOS> 
predictions = model.predict_classes(X_test, verbose=0)
 print(predictions)
 pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) , "Label": predictions} ).to_csv("preds.csv", index=False, header=True)