Upload folder using huggingface_hub

README.md

@@ -1,12 +1,6 @@
 ---
-title: LayoutLMv3 For Recepits
-emoji: 👁
-colorFrom: indigo
-colorTo: green
+title: LayoutLMv3_for_recepits
+app_file: app.py
 sdk: gradio
 sdk_version: 4.40.0
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,52 @@
+from cord_inference import prediction as cord_prediction
+from sroie_inference import prediction as sroie_prediction
+import gradio as gr
+import json
+
+def prediction(image_path: str):
+
+    #we first use mp-02/layoutlmv3-finetuned-cord on the image, which gives us a JSON with some info and a blurred image
+    d, image = sroie_prediction(image_path)
+
+    #we save the blurred image in order to pass it to the other model
+    image_path_blurred = image_path.split('.')[0] + '_blurred.' + image_path.split('.')[1]
+    image.save(image_path_blurred)
+
+    #then we use the model fine-tuned on sroie (for now it is Theivaprakasham/layoutlmv3-finetuned-sroie)
+    d1, image1 = cord_prediction(image_path_blurred)
+
+    #we then link the two json files
+    if len(d) == 0:
+        k = d1
+    else:
+        k = json.dumps(d).split('}')[0] + ', ' + json.dumps(d1).split('{')[1]
+
+    return d, image, d1, image1, k
+
+# p,i,j = prediction("11990982-img.png")
+# print(p)
+
+
+title = "Interactive demo: LayoutLMv3 for receipts"
+description = "Demo for Microsoft's LayoutLMv3, a Transformer for state-of-the-art document image understanding tasks. This particular model is fine-tuned on CORD and SROIE, which are datasets of receipts.\n It firsts uses the fine-tune on SROIE to extract date, company and address, then the fine-tune on CORD for the other info.\n To use it, simply upload an image or use the example image below. Results will show up in a few seconds."
+examples = [['image.jpg']]
+
+css = """.output_image, .input_image {height: 600px !important}"""
+
+
+# we use a gradio interface that takes in input an image and return a JSON file that contains its info
+# we show also the intermediate steps (first we take some info with the model fine-tuned on SROIE and we blur the relative boxes
+# then we pass the image to the model fine-tuned on CORD
+iface = gr.Interface(fn=prediction,
+                     inputs=gr.Image(type="filepath"),
+                     outputs=[gr.JSON(label="json parsing"),
+                              gr.Image(type="pil", label="blurred image"),
+                              gr.JSON(label="json parsing"),
+                              gr.Image(type="pil", label="annotated image"),
+                              gr.JSON(label="json parsing")],
+                     title=title,
+                     description=description,
+                     examples=examples,
+                     css=css)
+
+iface.launch()

cord_inference.py

@@ -0,0 +1,81 @@
+import torch
+import numpy as np
+from transformers import LayoutLMv3TokenizerFast, LayoutLMv3Processor, LayoutLMv3ForTokenClassification
+from PIL import Image, ImageDraw, ImageFont
+from utils import OCR, unnormalize_box
+
+
+labels = ["O", "B-MENU.NM", "B-MENU.NUM", "B-MENU.UNITPRICE", "B-MENU.CNT", "B-MENU.DISCOUNTPRICE", "B-MENU.PRICE", "B-MENU.ITEMSUBTOTAL", "B-MENU.VATYN", "B-MENU.ETC", "B-MENU.SUB.NM", "B-MENU.SUB.UNITPRICE", "B-MENU.SUB.CNT", "B-MENU.SUB.PRICE", "B-MENU.SUB.ETC", "B-VOID_MENU.NM", "B-VOID_MENU.PRICE", "B-SUB_TOTAL.SUBTOTAL_PRICE", "B-SUB_TOTAL.DISCOUNT_PRICE", "B-SUB_TOTAL.SERVICE_PRICE", "B-SUB_TOTAL.OTHERSVC_PRICE", "B-SUB_TOTAL.TAX_PRICE", "B-SUB_TOTAL.ETC", "B-TOTAL.TOTAL_PRICE", "B-TOTAL.TOTAL_ETC", "B-TOTAL.CASHPRICE", "B-TOTAL.CHANGEPRICE", "B-TOTAL.CREDITCARDPRICE", "B-TOTAL.EMONEYPRICE", "B-TOTAL.MENUTYPE_CNT", "B-TOTAL.MENUQTY_CNT", "I-MENU.NM", "I-MENU.NUM", "I-MENU.UNITPRICE", "I-MENU.CNT", "I-MENU.DISCOUNTPRICE", "I-MENU.PRICE", "I-MENU.ITEMSUBTOTAL", "I-MENU.VATYN", "I-MENU.ETC", "I-MENU.SUB.NM", "I-MENU.SUB.UNITPRICE", "I-MENU.SUB.CNT", "I-MENU.SUB.PRICE", "I-MENU.SUB.ETC", "I-VOID_MENU.NM", "I-VOID_MENU.PRICE", "I-SUB_TOTAL.SUBTOTAL_PRICE", "I-SUB_TOTAL.DISCOUNT_PRICE", "I-SUB_TOTAL.SERVICE_PRICE", "I-SUB_TOTAL.OTHERSVC_PRICE", "I-SUB_TOTAL.TAX_PRICE", "I-SUB_TOTAL.ETC", "I-TOTAL.TOTAL_PRICE", "I-TOTAL.TOTAL_ETC", "I-TOTAL.CASHPRICE", "I-TOTAL.CHANGEPRICE", "I-TOTAL.CREDITCARDPRICE", "I-TOTAL.EMONEYPRICE", "I-TOTAL.MENUTYPE_CNT", "I-TOTAL.MENUQTY_CNT"]
+id2label = {v: k for v, k in enumerate(labels)}
+label2id = {k: v for v, k in enumerate(labels)}
+
+tokenizer = LayoutLMv3TokenizerFast.from_pretrained("mp-02/layoutlmv3-finetuned-cord", apply_ocr=False)
+processor = LayoutLMv3Processor.from_pretrained("mp-02/layoutlmv3-finetuned-cord", apply_ocr=False)
+model = LayoutLMv3ForTokenClassification.from_pretrained("mp-02/layoutlmv3-finetuned-cord")
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model.to(device)
+
+
+def prediction(image_path: str):
+    image = Image.open(image_path).convert('RGB')
+    boxes, words = OCR(image_path)
+    encoding = processor(image, words, boxes=boxes, return_offsets_mapping=True, return_tensors="pt", truncation=True)
+    offset_mapping = encoding.pop('offset_mapping')
+
+    for k, v in encoding.items():
+        encoding[k] = v.to(device)
+
+    outputs = model(**encoding)
+
+    predictions = outputs.logits.argmax(-1).squeeze().tolist()
+    token_boxes = encoding.bbox.squeeze().tolist()
+
+    inp_ids = encoding.input_ids.squeeze().tolist()
+    inp_words = [tokenizer.decode(i) for i in inp_ids]
+
+    width, height = image.size
+    is_subword = np.array(offset_mapping.squeeze().tolist())[:, 0] != 0
+
+    true_predictions = [id2label[pred] for idx, pred in enumerate(predictions) if not is_subword[idx]]
+    true_boxes = [unnormalize_box(box, width, height) for idx, box in enumerate(token_boxes) if not is_subword[idx]]
+    true_words = []
+
+    for id, i in enumerate(inp_words):
+        if not is_subword[id]:
+            true_words.append(i)
+        else:
+            true_words[-1] = true_words[-1]+i
+
+    true_predictions = true_predictions[1:-1]
+    true_boxes = true_boxes[1:-1]
+    true_words = true_words[1:-1]
+
+    preds = []
+    l_words = []
+    bboxes = []
+
+    for i, j in enumerate(true_predictions):
+        if j != 'others':
+            preds.append(true_predictions[i])
+            l_words.append(true_words[i])
+            bboxes.append(true_boxes[i])
+
+    d = {}
+    for id, i in enumerate(preds):
+        if i not in d.keys():
+            d[i] = l_words[id]
+        else:
+            d[i] = d[i] + ", " + l_words[id]
+    d = {k: v.strip() for (k, v) in d.items()}
+
+    # TODO: process the json
+
+    draw = ImageDraw.Draw(image, "RGBA")
+    font = ImageFont.load_default()
+
+    for prediction, box in zip(preds, bboxes):
+        draw.rectangle(box)
+        draw.text((box[0]+10, box[1]-10), text=prediction, font=font, fill="black")
+
+    return d, image

requirements.txt

@@ -0,0 +1,6 @@
+json
+torch
+cv2
+PIL
+transformers
+paddleocr

sroie_inference.py

@@ -0,0 +1,114 @@
+import torch
+import cv2
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont
+from transformers import LayoutLMv3TokenizerFast, LayoutLMv3Processor, LayoutLMv3ForTokenClassification
+from utils import OCR, unnormalize_box
+
+
+labels = ["O", "B-COMPANY", "I-COMPANY", "B-DATE", "I-DATE", "B-ADDRESS", "I-ADDRESS", "B-TOTAL", "I-TOTAL"]
+id2label = {v: k for v, k in enumerate(labels)}
+label2id = {k: v for v, k in enumerate(labels)}
+
+tokenizer = LayoutLMv3TokenizerFast.from_pretrained("Theivaprakasham/layoutlmv3-finetuned-sroie", apply_ocr=False)
+processor = LayoutLMv3Processor.from_pretrained("Theivaprakasham/layoutlmv3-finetuned-sroie", apply_ocr=False)
+model = LayoutLMv3ForTokenClassification.from_pretrained("Theivaprakasham/layoutlmv3-finetuned-sroie")
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model.to(device)
+
+
+def blur(image, boxes):
+    img = cv2.imread(image)
+    for box in boxes:
+        blur_x = int(box[0])
+        blur_y = int(box[1])
+        blur_width = int(box[2]-box[0])
+        blur_height = int(box[3]-box[1])
+
+        roi = img[blur_y:blur_y + blur_height, blur_x:blur_x + blur_width]
+        blur_image = cv2.GaussianBlur(roi, (201, 201), 0)
+        img[blur_y:blur_y + blur_height, blur_x:blur_x + blur_width] = blur_image
+
+    cv2.imwrite("images/example_with_blur.jpg", img)
+    return "example_with_blur.jpg"
+
+
+def prediction(image_path: str):
+    boxes, words = OCR(image_path)
+    image = Image.open(image_path).convert('RGB')
+    encoding = processor(image, words, boxes=boxes, return_offsets_mapping=True, return_tensors="pt", truncation=True)
+    offset_mapping = encoding.pop('offset_mapping')
+
+    for k, v in encoding.items():
+        encoding[k] = v.to(device)
+
+    outputs = model(**encoding)
+
+    predictions = outputs.logits.argmax(-1).squeeze().tolist()
+    token_boxes = encoding.bbox.squeeze().tolist()
+
+    inp_ids = encoding.input_ids.squeeze().tolist()
+    inp_words = [tokenizer.decode(i) for i in inp_ids]
+
+    width, height = image.size
+    is_subword = np.array(offset_mapping.squeeze().tolist())[:, 0] != 0
+
+    true_predictions = [id2label[pred] for idx, pred in enumerate(predictions) if not is_subword[idx]]
+    true_boxes = [unnormalize_box(box, width, height) for idx, box in enumerate(token_boxes) if not is_subword[idx]]
+    true_words = []
+
+    for id, i in enumerate(inp_words):
+        if not is_subword[id]:
+            true_words.append(i)
+        else:
+            true_words[-1] = true_words[-1]+i
+
+    true_predictions = true_predictions[1:-1]
+    true_boxes = true_boxes[1:-1]
+    true_words = true_words[1:-1]
+
+    preds = []
+    l_words = []
+    bboxes = []
+
+    for i, j in enumerate(true_predictions):
+        if j != 'others':
+            preds.append(true_predictions[i])
+            l_words.append(true_words[i])
+            bboxes.append(true_boxes[i])
+
+    d = {}
+    for id, i in enumerate(preds):
+        if i not in d.keys():
+            d[i] = l_words[id]
+        else:
+            d[i] = d[i] + ", " + l_words[id]
+
+    d = {k: v.strip() for (k, v) in d.items()}
+
+    keys_to_pop = []
+    for k, v in d.items():
+        if k[:2] == "I-":
+            d["B-" + k[2:]] = d["B-" + k[2:]] + ", " + v
+            keys_to_pop.append(k)
+
+    if "O" in d: d.pop("O")
+    if "B-TOTAL" in d: d.pop("B-TOTAL")
+    for k in keys_to_pop: d.pop(k)
+
+    blur_boxes = []
+    for prediction, box in zip(preds, bboxes):
+        if prediction != 'O' and prediction[2:] != 'TOTAL':
+            blur_boxes.append(box)
+
+    image = Image.open(blur(image_path, blur_boxes))
+
+    draw = ImageDraw.Draw(image, "RGBA")
+    font = ImageFont.load_default()
+    for prediction, box in zip(preds, bboxes):
+        draw.rectangle(box)
+        draw.text((box[0]+10, box[1]-10), text=prediction, font=font, fill="black", font_size="8")
+
+    return d, image
+

utils.py

@@ -0,0 +1,40 @@
+from paddleocr import PaddleOCR
+from PIL import Image
+
+def normalize_bbox(bbox, width, height):
+
+    return [
+        int(1000 * (bbox[0] / width)),
+        int(1000 * (bbox[1] / height)),
+        int(1000 * (bbox[2] / width)),
+        int(1000 * (bbox[3] / height)),
+    ]
+
+def unnormalize_box(bbox, width, height):
+
+    return [
+        width * (bbox[0] / 1000),
+        height * (bbox[1] / 1000),
+        width * (bbox[2] / 1000),
+        height * (bbox[3] / 1000),
+    ]
+
+
+def OCR(image_path: str):
+    ocr = PaddleOCR(use_angle_cls=True)
+    image = Image.open(image_path)
+    result = ocr.ocr(image_path, cls=True)
+    bboxes = []
+    words = []
+
+    for idx in range(len(result)):
+        res = result[idx]
+
+        for line in res:
+            # print(line)
+            # print(line[0][0] + line[0][2])
+            bboxes.append(normalize_bbox(line[0][0]+line[0][2], image.width, image.height))
+            # print(line[1][0])
+            words.append(line[1][0])
+
+    return bboxes, words