I builded an model to predict handwritten word but it it does not work well this is my code can you give some improvement: 
import numpy as np
import pandas as pd
import os
import cv2
import imutils
import random
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from sklearn.preprocessing import LabelBinarizer
from keras.layers import Flatten, Dense,Dropout
from keras.layers import Conv2D, MaxPooling2D

dir = "Final-state-Total"
dir2 = "dataset1"
df = pd.read_csv(os.path.join(dir2,"english.csv"))
df['label'] = df['label'].str.upper()
data = []
img_size = 32
non_chars = ["#","$","&","@"]
for i in os.listdir(dir):
    if i in non_chars:
        continue
    listeToAdd = df[ df['label'] == i]
    listeToAdd = listeToAdd['image'].tolist()
    for j in listeToAdd:
        img = cv2.imread(os.path.join(dir2,j),0)
        img = cv2.resize(img,(img_size,img_size))
        data.append([img,i])
    count = 0
    sub_directory = os.path.join(dir,i)
    for j in os.listdir(sub_directory):
        count+=1
        if count > 4000:
            break
        img = cv2.imread(os.path.join(sub_directory,j),0)
        img = cv2.resize(img,(img_size,img_size))
        data.append([img,i]) 

random.shuffle(data)
len(data) # 21174
dataX = []
dataY = []
for feautures,label in data:
    dataX.append(feautures)
    dataY.append(label)
nb_classes = len(np.unique(dataY)) # 64 classes 
LB = LabelBinarizer()
dataY = LB.fit_transform(dataY)
dataX = np.array(dataX)/255.0
dataX = dataX.reshape(-1,img_size,img_size,1)
X_train, X_val, y_train, y_val = train_test_split(dataX, dataY, test_size=0.2, random_state=42) # (16939, 32, 32, 1) (4235, 32, 32, 1) (16939, 64) (4235, 64) 
model = Sequential()
model.add(Conv2D(32, (3, 3), padding = "same", activation='relu', input_shape=(32,32,1)))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(64, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train,y_train, epochs=50, batch_size=32, validation_data = (X_val, y_val),  verbose=1)
def sort_contours(cnts, method="left-to-right"):
    reverse = False
    i = 0
    if method == "right-to-left" or method == "bottom-to-top":
        reverse = True
    if method == "top-to-bottom" or method == "bottom-to-top":
        i = 1
    boundingBoxes = [cv2.boundingRect(c) for c in cnts]
    (cnts, boundingBoxes) = zip(*sorted(zip(cnts, boundingBoxes),
    key=lambda b:b[1][i], reverse=reverse))
    # return the list of sorted contours and bounding boxes
    return (cnts, boundingBoxes)
def get_letters(img):
    letters = []
    image = cv2.imread(img)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    ret,thresh1 = cv2.threshold(gray ,127,255,cv2.THRESH_BINARY_INV)
    dilated = cv2.dilate(thresh1, None, iterations=2)

    cnts = cv2.findContours(dilated.copy(), cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    cnts = imutils.grab_contours(cnts)
    cnts = sort_contours(cnts, method="left-to-right")[0]
    # loop over the contours
    for c in cnts:
        if cv2.contourArea(c) > 10:
            (x, y, w, h) = cv2.boundingRect(c)
            cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
        roi = gray[y:y + h, x:x + w]
        thresh = cv2.threshold(roi, 0, 255,cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]
        thresh = cv2.resize(thresh, (32, 32), interpolation = cv2.INTER_CUBIC)
        thresh = thresh.astype("float32") / 255.0
        thresh = np.expand_dims(thresh, axis=-1)
        thresh = thresh.reshape(1,32,32,1)
        ypred = model.predict(thresh)
        ypred = LB.inverse_transform(ypred)
        [x] = ypred
        letters.append(x)
    return letters, image
def get_word(letter):
    word = "".join(letter)
    return word
letter,image = get_letters("brother.jpg")
word = get_word(letter)
print(word)
plt.imshow(image) # what it supposed to give BROTHER ## what give FFQFFFF