# from __future__ import division, print_function # coding=utf-8 # import sys import os # import glob # import re import numpy as np # import datetime # Keras # from tensorflow.keras.models import load_model #from tensorflow.keras.preprocessing import image # Flask utils # from flask import Flask, redirect, url_for, request, render_template # from werkzeug.utils import secure_filename # from gevent.pywsgi import WSGIServer #import everytnimg # from skimage.io import imread, imshow # from skimage.filters import gaussian, threshold_otsu # from skimage.feature import canny # from skimage.transform import probabilistic_hough_line, rotate # from process_image import process_image # import glob # import math import cv2 # import numpy as np # from PIL import Image # from matplotlib import pyplot as plt # from matplotlib.patches import Rectangle #%matplotlib inline # from collections import OrderedDict # from PIL import Image # import pandas as pd # import seaborn as sns # import math #import all from Hough transfrom cell # from skimage.transform import hough_line, hough_line_peaks # from skimage.transform import rotate # from skimage.feature import canny # from skimage.io import imread # from skimage.color import rgb2gray # import matplotlib.pyplot as plt # from scipy.stats import mode as md # from myhough import deskew, deskew2 # from segment_words import sortit,words,createk,hpf,bps,wps,baw # from myverify import verify #from detect_frame import detect_frame # import pathlib #import more import tensorflow as tf from object_detection.utils import config_util # from object_detection.protos import pipeline_pb2 # from google.protobuf import text_format # import os from object_detection.utils import label_map_util # from object_detection.utils import visualization_utils as viz_utils from object_detection.builders import model_builder # Load pipeline config and build a detection model WORKSPACE_PATH = 'Tensorflow/workspace' # SCRIPTS_PATH = 'Tensorflow/scripts' #APIMODEL_PATH = 'Tensorflow/models' ANNOTATION_PATH = WORKSPACE_PATH+'/annotations' # IMAGE_PATH = WORKSPACE_PATH+'/images' MODEL_PATH = WORKSPACE_PATH+'/models' PRETRAINED_MODEL_PATH = WORKSPACE_PATH+'/pre-trained-models' CONFIG_PATH = MODEL_PATH+'/my_ssd_mobnet/pipeline.config' CHECKPOINT_PATH = MODEL_PATH+'/my_ssd_mobnet/' # INPUT_IMAGE_PATH = 'Tensorflow/myimages' # MODEL_PATH = 'E:/RealTimeObjectDetection/model.best.hdf5' configs = config_util.get_configs_from_pipeline_file(CONFIG_PATH) detection_model = model_builder.build(model_config=configs['model'], is_training=False) # Restore checkpoint ckpt = tf.compat.v2.train.Checkpoint(model=detection_model) ckpt.restore(os.path.join(CHECKPOINT_PATH, 'ckpt-51')).expect_partial() @tf.function def detect_fn(image): image, shapes = detection_model.preprocess(image) prediction_dict = detection_model.predict(image, shapes) detections = detection_model.postprocess(prediction_dict, shapes) return detections def detect_frame(frame,isRealTime = False): image_np = np.array(frame) cpimg = frame.copy() input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32) detections = detect_fn(input_tensor) print(len(detections)) num_detections = int(detections.pop('num_detections')) #print("hello") #print(num_detections) #print(len(detections['detection_scores'])) detections = {key: value[0, :num_detections].numpy() for key, value in detections.items()} detections['num_detections'] = num_detections row,col,dummy = image_np.shape # detection_classes should be ints. detections['detection_classes'] = detections['detection_classes'].astype(np.int64) #print(detections['detection_classes']) mark = [0]*15 myletters = [] for i in range(0,15): curi=detections['detection_classes'][i] classi=classes[curi] print(classes[curi],end='-') cur=detections['detection_scores'][i] if(cur<0.2): continue print(cur,end=' ') print(detections['detection_boxes'][i], end=' ') x0=(detections['detection_boxes'][i][0]) y0=(detections['detection_boxes'][i][1]) x1=(detections['detection_boxes'][i][2]) y1=(detections['detection_boxes'][i][3]) curarea=(x1-x0)*(y1-y0) ok=1 for j in range(0,i): #print(mark[j]) if mark[j]==0: continue curj=detections['detection_classes'][j] classj=classes[curj] if classi=='ি' or classj=='ি': if classi!=classj: continue if classi=='ী' or classj=='ী': if classi!=classj: continue x2=(detections['detection_boxes'][j][0]) y2=(detections['detection_boxes'][j][1]) x3=(detections['detection_boxes'][j][2]) y3=(detections['detection_boxes'][j][3]) x4=max(x0,x2) y4=max(y0,y2) x5=min(x1,x3) y5=min(y1,y3) if x4>x5 or y4>y5: continue prevarea=(x3-x2)*(y3-y2) commonarea=(x5-x4)*(y5-y4) ins1=curarea/commonarea ins2=prevarea/commonarea ins=commonarea/(curarea+prevarea-commonarea) print(ins1,end=' ') if(ins>=0.5): ok=0 cur=detections['detection_classes'][j] print(classes[cur]) break if ok==1: mark[i]=1 cur=detections['detection_classes'][i] #myletters.append(classes[cur]) print(ok) #verification for i in range(0,15): if mark[i]==0 or avver==0: continue if detections['detection_classes'][i]>38: continue x0=int(detections['detection_boxes'][i][0]*row) y0=int(detections['detection_boxes'][i][1]*col) x1=int(detections['detection_boxes'][i][2]*row) y1=int(detections['detection_boxes'][i][3]*col) #print(y0,y1,x0,x1) currImg = cpimg[x0:x1,y0:y1] curscore = detections['detection_scores'][i] curclass = detections['detection_classes'][i] label,conf = verify(currImg) #print(ulta[label],conf) #print(curclass,curscore) if conf>curscore and ulta[label]!=curclass and ulta[label]!=-1: detections['detection_classes'][i]=ulta[label] detections['detection_scores'][i]=conf for i in range(0,15): if(detections['detection_scores'][i]<0.2): continue if mark[i]==0: continue cur=detections['detection_classes'][i] cur=classes[cur] y0=(detections['detection_boxes'][i][1]) y1=(detections['detection_boxes'][i][3]) pair = (y0,cur,y1) myletters.append(pair) myletters.sort(key = lambda x: x[0]) #print(myletters) for i in range(len(myletters)-1,-1,-1): y0=myletters[i][0] curr=myletters[i][1] y1=myletters[i][2] if curr=='ু' or curr=='্র': mxarea=0 mxno=i-1 for j in range(0,len(myletters)): if i==j: continue y2=myletters[j][0] y3=myletters[j][2] curcommon = min(y3,y1)-max(y0,y2) if curcommon>mxarea: mxarea = curcommon mxno=j if mxno!=(i-1): myletters[i],myletters[i+1]=myletters[i+1],myletters[i] res_list = [x[1] for x in myletters] print(res_list) for i in range(len(res_list)-2, -1, -1): x=res_list[i] y=res_list[i+1] if x=='ে' or x=='ি': res_list[i],res_list[i+1]=res_list[i+1],res_list[i] for i in range(len(res_list)-2, -1, -1): x=res_list[i] y=res_list[i+1] print(x,y) if x=='অ' and y=='া': print('yo') res_list[i]='আ' res_list.pop(i+1) print(res_list) for i in res_list: print(i,end='') print(' ') return res_list # Define a flask app # app = Flask(__name__) # Model saved with Keras model.save() # Load your trained model # model = load_model(MODEL_PATH) #model._make_predict_function() # Necessary # print('Model loaded. Start serving...') # You can also use pretrained model from Keras # Check https://keras.io/applications/ #from keras.applications.resnet50 import ResNet50 #model = ResNet50(weights='imagenet') #model.save('') # print('Model loaded. Check http://127.0.0.1:5000/') avver=0 clicked=1 wp = None; bp = None; category_index = label_map_util.create_category_index_from_labelmap(ANNOTATION_PATH+'/label_map.pbtxt') classes=['অ','ই','উ','এ','ও','ক','খ','গ','ঘ','চ','ছ','জ','ঝ','ট','ঠ','ড','ত','থ','দ','ধ','ন','প','ফ','ব','ভ','ম','য','র','ল','শ','ষ','স','হ','ড়','য়','ৎ','ং','ঁ','০','১','২','৩','৪','৫','৭','৮','া','ি','ী','ে','ু','্র','্য'] labels=[1,2,4,7,9,11,12,13,14,16,17,18,19,21,22,23,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,45,46,47,49,50,51,52,53,54,55,57,58,60,61,62,63,64,66,67] ulta=[0,-1,1,-1,2,-1,-1,3,-1,4,-1,5,6,7,8,-1,9,10,11,12,-1,13,14,15,-1,-1,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,-1,34,35,36,-1,37,38,39,40,41,42,43,-1,44,45,-1,46,47,48,49,50,-1,51,52] def model_predict(word): #img = cv2.imread(img_path,cv2.IMREAD_GRAYSCALE) ''' if clicked==1: bp = 66 wp = 160 mode = "GCMODE" if mode == "GCMODE": img= hpf(img,kSize = 51) wp = 127 img = wps(img,wp) img = bps(img) elif mode == "RMODE": bps() wps() elif mode == "SMODE": bps() wps() baw() img = cv2.fastNlMeansDenoising(img, img, 50.0, 7, 21) print("\ndone.") xs=img.shape if len(xs)==3: img = img[:,:,0] img = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C,cv.THRESH_BINARY,11,2) angeel = deskew(img) if angeel!=0: img = deskew2(img,angeel) ho,wo=img.shape area=ho*wo ara=words(img,25,11,7,area/5000) ara.reverse() #cv2.imshow('input image',img) sz=len(ara) for i in range(0,sz): ara[i]=sorted(ara[i], key=lambda entry:entry[0][0]) cnt2=0 files = glob.glob('Tensorflow/myimages/*') for f in files: os.remove(f) for i in range(0,sz): #print(ara[i].shape) tmp=ara[i] sz2=len(tmp) if i%10==0: cnt2=cnt2+1 for j in range(0,sz2): a,b=tmp[j] b = cv2.adaptiveThreshold(b,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2) if j<10: cnt3=0 elif j<20: cnt3=1 else: cnt3=2 cv2.imwrite('Tensorflow/myimages/ocr %d%d%d%d.jpg' % (cnt2,i,cnt3,j), b) #cv2.imshow('Crop %d%d' % (i,j), b) cv2.waitKey(0) PATH_TO_TEST_IMAGES_DIR = pathlib.Path('Tensorflow/myimages') TEST_IMAGE_PATHS = (list(PATH_TO_TEST_IMAGES_DIR.glob("*.jpg"))+list(PATH_TO_TEST_IMAGES_DIR.glob("*.jpeg"))) #+list(PATH_TO_TEST_IMAGES_DIR.glob("*.png")) print(len(TEST_IMAGE_PATHS)) final = [] for image_path in TEST_IMAGE_PATHS: print("ovi") print(image_path) frame = cv2.imread(str(image_path)) x=str(image_path) print(x[25]) # gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) final.append((detect_frame(frame),x[25])) ''' frame = cv2.fastNlMeansDenoising(word,word, 50.0, 7, 21) xs = frame.shape if(len(xs)==3): frame = frame[:,:,0] frame= cv2.adaptiveThreshold(frame,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2) frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR) # x=str(img_path) #print(x[25]) # gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) image_np = np.array(frame) cpimg = frame.copy() input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32) image_t, shapes = detection_model.preprocess(input_tensor) prediction_dict = detection_model.predict(image_t, shapes) detections = detection_model.postprocess(prediction_dict, shapes) # print(len(detections)) num_detections = int(detections.pop('num_detections')) #print("hello") #print(num_detections) #print(len(detections['detection_scores'])) detections = {key: value[0, :num_detections].numpy() for key, value in detections.items()} detections['num_detections'] = num_detections row,col,dummy = image_np.shape # detection_classes should be ints. detections['detection_classes'] = detections['detection_classes'].astype(np.int64) #print(detections['detection_classes']) mark = [0]*15 myletters = [] for i in range(0,15): curi=detections['detection_classes'][i] classi=classes[curi] # print(classes[curi],end='-') cur=detections['detection_scores'][i] if(cur<0.2): continue # print(cur,end=' ') # print(detections['detection_boxes'][i], end=' ') x0=(detections['detection_boxes'][i][0]) y0=(detections['detection_boxes'][i][1]) x1=(detections['detection_boxes'][i][2]) y1=(detections['detection_boxes'][i][3]) curarea=(x1-x0)*(y1-y0) ok=1 for j in range(0,i): #print(mark[j]) if mark[j]==0: continue curj=detections['detection_classes'][j] classj=classes[curj] if classi=='ি' or classj=='ি': if classi!=classj: continue if classi=='ী' or classj=='ী': if classi!=classj: continue x2=(detections['detection_boxes'][j][0]) y2=(detections['detection_boxes'][j][1]) x3=(detections['detection_boxes'][j][2]) y3=(detections['detection_boxes'][j][3]) x4=max(x0,x2) y4=max(y0,y2) x5=min(x1,x3) y5=min(y1,y3) if x4>x5 or y4>y5: continue prevarea=(x3-x2)*(y3-y2) commonarea=(x5-x4)*(y5-y4) ins1=curarea/commonarea ins2=prevarea/commonarea ins=commonarea/(curarea+prevarea-commonarea) # print(ins1,end=' ') if(ins>=0.5): ok=0 cur=detections['detection_classes'][j] # print(classes[cur]) break if ok==1: mark[i]=1 cur=detections['detection_classes'][i] #myletters.append(classes[cur]) # print(ok) #verification for i in range(0,15): if mark[i]==0 or avver==0: continue if detections['detection_classes'][i]>38: continue x0=int(detections['detection_boxes'][i][0]*row) y0=int(detections['detection_boxes'][i][1]*col) x1=int(detections['detection_boxes'][i][2]*row) y1=int(detections['detection_boxes'][i][3]*col) #print(y0,y1,x0,x1) currImg = cpimg[x0:x1,y0:y1] curscore = detections['detection_scores'][i] curclass = detections['detection_classes'][i] label,conf = verify(currImg) #print(ulta[label],conf) #print(curclass,curscore) if conf>curscore and ulta[label]!=curclass and ulta[label]!=-1: detections['detection_classes'][i]=ulta[label] detections['detection_scores'][i]=conf for i in range(0,15): if(detections['detection_scores'][i]<0.2): continue if mark[i]==0: continue cur=detections['detection_classes'][i] cur=classes[cur] y0=(detections['detection_boxes'][i][1]) y1=(detections['detection_boxes'][i][3]) pair = (y0,cur,y1) myletters.append(pair) myletters.sort(key = lambda x: x[0]) #print(myletters) for i in range(len(myletters)-1,-1,-1): y0=myletters[i][0] curr=myletters[i][1] y1=myletters[i][2] if curr=='ু' or curr=='্র': mxarea=0 mxno=i-1 for j in range(0,len(myletters)): if i==j: continue y2=myletters[j][0] y3=myletters[j][2] curcommon = min(y3,y1)-max(y0,y2) if curcommon>mxarea: mxarea = curcommon mxno=j if mxno!=(i-1): myletters[i],myletters[i+1]=myletters[i+1],myletters[i] res_list = [x[1] for x in myletters] # print(res_list) for i in range(len(res_list)-2, -1, -1): x=res_list[i] y=res_list[i+1] if x=='ে' or x=='ি': res_list[i],res_list[i+1]=res_list[i+1],res_list[i] for i in range(len(res_list)-2, -1, -1): x=res_list[i] y=res_list[i+1] # print(x,y) if x=='অ' and y=='া': # print('yo') res_list[i]='আ' res_list.pop(i+1) # print(res_list) output='' for i in res_list: output=output+i # print(' ') # time_now = datetime.datetime.now().strftime('%m_%d_%Y_%I_%M_%S_%p') # # print(time_now) # date = datetime.date.today().strftime('%Y_%m_%d') # # print(date) # folderName = "created/"+date # if(not os.path.isdir(folderName)): # os.makedirs(folderName) # fileName = folderName+ "/" + time_now + ".png" # cv2.imwrite(fileName,word) return output ''' output='' for i in range(0,len(final)): ara=final[i][0] numb=final[i][1] if i>0 and numb!=final[i-1][1]: output= output+'\n' word = ''.join(ara) #corrected_word = get_campaign(word) output= output + word #print(corrected_word,end='') output = output + ' ' return output ''' import gradio as gr demo = gr.Interface(fn=model_predict, inputs= "paint", outputs="text") demo.launch()