Update app.py
Browse files
app.py
CHANGED
@@ -1,10 +1,3 @@
|
|
1 |
-
# import gradio as gr
|
2 |
-
|
3 |
-
# def greet(name):
|
4 |
-
# return "Hello " + name + "!!"
|
5 |
-
|
6 |
-
# iface = gr.Interface(fn=greet, inputs="text", outputs="text")
|
7 |
-
# iface.launch()
|
8 |
from transformers import TrOCRProcessor, VisionEncoderDecoderModel
|
9 |
from PIL import Image
|
10 |
import requests
|
@@ -19,6 +12,24 @@ import gradio as gr
|
|
19 |
from skimage.filters import threshold_otsu
|
20 |
from skimage.util import invert
|
21 |
import cv2,imageio
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
22 |
processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten')
|
23 |
model = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-handwritten')
|
24 |
plt.switch_backend('Agg')
|
@@ -305,7 +316,149 @@ def extract(image):
|
|
305 |
result=result+" "
|
306 |
return result
|
307 |
|
308 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
309 |
inputs=[gr.inputs.Image(type='filepath', label='Ideal Answer'),gr.inputs.Image(type='filepath', label='Ideal Answer Diagram'),gr.inputs.Image(type='filepath', label='Submitted Answer'),gr.inputs.Image(type='filepath', label='Submitted Answer Diagram')],
|
310 |
outputs=gr.outputs.Textbox(),)
|
311 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
from transformers import TrOCRProcessor, VisionEncoderDecoderModel
|
2 |
from PIL import Image
|
3 |
import requests
|
|
|
12 |
from skimage.filters import threshold_otsu
|
13 |
from skimage.util import invert
|
14 |
import cv2,imageio
|
15 |
+
from matplotlib.dates import SU
|
16 |
+
from regex import F
|
17 |
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
18 |
+
from sentence_transformers import SentenceTransformer, util
|
19 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
20 |
+
import spacy
|
21 |
+
import pandas as pd
|
22 |
+
from tqdm import tqdm
|
23 |
+
import textdistance
|
24 |
+
from spacy.lang.en.stop_words import STOP_WORDS
|
25 |
+
#import psycopg2
|
26 |
+
import os
|
27 |
+
rom tensorflow.keras.applications.resnet50 import ResNet50,preprocess_input, decode_predictions
|
28 |
+
from tensorflow.keras.preprocessing import image
|
29 |
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
30 |
+
|
31 |
+
|
32 |
+
|
33 |
processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten')
|
34 |
model = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-handwritten')
|
35 |
plt.switch_backend('Agg')
|
|
|
316 |
result=result+" "
|
317 |
return result
|
318 |
|
319 |
+
|
320 |
+
|
321 |
+
|
322 |
+
nlp = spacy.load("en_core_web_md")
|
323 |
+
|
324 |
+
|
325 |
+
def listToString(s):
|
326 |
+
|
327 |
+
# initialize an empty string
|
328 |
+
str1 = " "
|
329 |
+
|
330 |
+
# return string
|
331 |
+
return (str1.join(s))
|
332 |
+
|
333 |
+
def rm_stop(my_doc):
|
334 |
+
# Create list of word tokens
|
335 |
+
token_list = []
|
336 |
+
for token in my_doc:
|
337 |
+
token_list.append(token.text)
|
338 |
+
|
339 |
+
|
340 |
+
|
341 |
+
# Create list of word tokens after removing stopwords
|
342 |
+
filtered_sentence =[]
|
343 |
+
|
344 |
+
for word in token_list:
|
345 |
+
lexeme = nlp.vocab[word]
|
346 |
+
if lexeme.is_stop == False:
|
347 |
+
filtered_sentence.append(word)
|
348 |
+
|
349 |
+
return filtered_sentence
|
350 |
+
|
351 |
+
def text_processing(sentence):
|
352 |
+
|
353 |
+
sentence = [token.lemma_.lower()
|
354 |
+
for token in nlp(sentence)
|
355 |
+
if token.is_alpha and not token.is_stop]
|
356 |
+
|
357 |
+
return sentence
|
358 |
+
|
359 |
+
def jaccard_sim(sent1,sent2):
|
360 |
+
# Text Processing
|
361 |
+
sentence1 = text_processing(sent1)
|
362 |
+
sentence2 = text_processing(sent2)
|
363 |
+
|
364 |
+
# Jaccard similarity
|
365 |
+
return textdistance.jaccard.normalized_similarity(sentence1, sentence2)
|
366 |
+
|
367 |
+
def sim(Ideal_Answer,Submitted_Answer):
|
368 |
+
# SBERT EMBEDDINGS
|
369 |
+
text1=Ideal_Answer.replace("\"","").replace("\'","")
|
370 |
+
text2=Submitted_Answer.replace("\"","").replace("\'","")
|
371 |
+
output=[]
|
372 |
+
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
|
373 |
+
|
374 |
+
#Compute embedding for both lists
|
375 |
+
embedding_1= model.encode(text1, convert_to_tensor=True)
|
376 |
+
embedding_2 = model.encode(text2, convert_to_tensor=True)
|
377 |
+
|
378 |
+
score=util.pytorch_cos_sim(embedding_1, embedding_2)
|
379 |
+
output.append("SBERT:"+str(int(float(str(score).split("[")[2].split("]")[0])*10.0))+",")
|
380 |
+
sbert=int(float(str(score).split("[")[2].split("]")[0])*10.0)
|
381 |
+
#Jaccard
|
382 |
+
output.append("Jaccard:"+str(int(jaccard_sim(text1,text2)*10.0))+",")
|
383 |
+
|
384 |
+
#spacy average word2vec
|
385 |
+
nlp = spacy.load("en_core_web_md") # make sure to use larger package!
|
386 |
+
doc1 = listToString(rm_stop(nlp(text1)))
|
387 |
+
doc2 = listToString(rm_stop(nlp(text2)))
|
388 |
+
|
389 |
+
# Similarity of two documents
|
390 |
+
w2v=int(nlp(doc1).similarity(nlp(doc2))*10.0)
|
391 |
+
final_score=int(0.8*sbert+0.2*w2v)
|
392 |
+
output.append("Word2Vec:"+str(int(nlp(doc1).similarity(nlp(doc2))*10.0))+",final_score:"+str(final_score))
|
393 |
+
out_string=listToString(output)
|
394 |
+
#return out_string
|
395 |
+
return str(out_string),final_score
|
396 |
+
|
397 |
+
|
398 |
+
|
399 |
+
def return_image_embedding(model,img_path):
|
400 |
+
img = image.load_img(img_path, target_size=(224, 224))
|
401 |
+
x = image.img_to_array(img)
|
402 |
+
x = np.expand_dims(x, axis=0)
|
403 |
+
x = preprocess_input(x)
|
404 |
+
preds = model.predict(x)
|
405 |
+
curr_df = pd.DataFrame(preds[0]).T
|
406 |
+
return curr_df
|
407 |
+
|
408 |
+
|
409 |
+
|
410 |
+
def draw_boxes(image, bounds, color='yellow', width=2):
|
411 |
+
draw = ImageDraw.Draw(image)
|
412 |
+
for bound in bounds:
|
413 |
+
p0, p1, p2, p3 = bound[0]
|
414 |
+
draw.line([*p0, *p1, *p2, *p3, *p0], fill=color, width=width)
|
415 |
+
return image
|
416 |
+
|
417 |
+
def inference(img, lang):
|
418 |
+
reader = easyocr.Reader(lang)
|
419 |
+
bounds = reader.readtext(img.name)
|
420 |
+
im = PIL.Image.open(img.name)
|
421 |
+
draw_boxes(im, bounds)
|
422 |
+
im.save('result.jpg')
|
423 |
+
return ['result.jpg', pd.DataFrame(bounds).iloc[: , 1:]]
|
424 |
+
|
425 |
+
def compute_tfidf_embeddings(documents1, documents2):
|
426 |
+
# Combine both lists of words into a single list
|
427 |
+
combined_documents = documents1 + documents2
|
428 |
+
|
429 |
+
# Initialize the TF-IDF vectorizer
|
430 |
+
vectorizer = TfidfVectorizer()
|
431 |
+
|
432 |
+
# Fit the vectorizer on the combined documents
|
433 |
+
vectorizer.fit(combined_documents)
|
434 |
+
|
435 |
+
# Transform the documents to TF-IDF embeddings
|
436 |
+
embeddings1 = vectorizer.transform(documents1)
|
437 |
+
embeddings2 = vectorizer.transform(documents2)
|
438 |
+
|
439 |
+
return embeddings1, embeddings2
|
440 |
+
|
441 |
+
|
442 |
+
def extract_eval(image1,image2,image3,image4):
|
443 |
+
ideal_text=extract(image1)
|
444 |
+
print("Extracting Ideal Text \n")
|
445 |
+
print(ideal_text)
|
446 |
+
submitted_text=extract(image3)
|
447 |
+
print("Extracting Submitted Text \n")
|
448 |
+
print(submitted_text_text)
|
449 |
+
a,b=sim(ideal_text,submitted_text)
|
450 |
+
print(a)
|
451 |
+
text_sim_score=b
|
452 |
+
model = ResNet50(include_top=False, weights='imagenet', pooling='avg')
|
453 |
+
diagram_1_embed=return_image_embedding(model,image2)
|
454 |
+
diagram_2_embed=return_image_embedding(model,image4)
|
455 |
+
diagram_embed_sim_score=util.pytorch_cos_sim(embedding_1, embedding_2)
|
456 |
+
print("Diagram Embedding Similarity Score \n")
|
457 |
+
print(diagram_embed_sim_score)
|
458 |
+
|
459 |
+
|
460 |
+
|
461 |
+
iface = gr.Interface(fn=extract_eval,
|
462 |
inputs=[gr.inputs.Image(type='filepath', label='Ideal Answer'),gr.inputs.Image(type='filepath', label='Ideal Answer Diagram'),gr.inputs.Image(type='filepath', label='Submitted Answer'),gr.inputs.Image(type='filepath', label='Submitted Answer Diagram')],
|
463 |
outputs=gr.outputs.Textbox(),)
|
464 |
|