truncation = true

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

README.md

@@ -1,3 +1,182 @@
 ---
-license: mit
+tags:
+- generated_from_trainer
+- endpoints-template
+library_name: generic
+datasets:
+- funsd
+model-index:
+- name: layoutlm-funsd
+  results: []
+pipeline_tag: other
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# layoutlm-funsd
+
+This model is a fine-tuned version of [microsoft/layoutlm-base-uncased](https://huggingface.co/microsoft/layoutlm-base-uncased) on the funsd dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.0045
+- Answer: {'precision': 0.7348314606741573, 'recall': 0.8084054388133498, 'f1': 0.7698646262507357, 'number': 809}
+- Header: {'precision': 0.44285714285714284, 'recall': 0.5210084033613446, 'f1': 0.47876447876447875, 'number': 119}
+- Question: {'precision': 0.8211009174311926, 'recall': 0.8403755868544601, 'f1': 0.8306264501160092, 'number': 1065}
+- Overall Precision: 0.7599
+- Overall Recall: 0.8083
+- Overall F1: 0.7866
+- Overall Accuracy: 0.8106
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 15
+- mixed_precision_training: Native AMP
+
+## Deploy Model with Inference Endpoints
+
+Before we can get started, make sure you meet all of the following requirements:
+
+1. An Organization/User with an active plan and *WRITE* access to the model repository.
+2. Can access the UI: [https://ui.endpoints.huggingface.co](https://ui.endpoints.huggingface.co/endpoints)
+
+
+
+### 1. Deploy LayoutLM and Send requests
+
+In this tutorial, you will learn how to deploy a [LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm) to [Hugging Face Inference Endpoints](https://huggingface.co/inference-endpoints) and how you can integrate it via an API into your products. 
+
+This tutorial is not covering how you create the custom handler for inference. If you want to learn how to create a custom Handler for Inference Endpoints, you can either checkout the [documentation](https://huggingface.co/docs/inference-endpoints/guides/custom_handler) or go through [“Custom Inference with Hugging Face Inference Endpoints”](https://www.philschmid.de/custom-inference-handler) 
+
+We are going to deploy [philschmid/layoutlm-funsd](https://huggingface.co/philschmid/layoutlm-funsd) which implements the following `handler.py` 
+
+```python
+from typing import Dict, List, Any
+from transformers import LayoutLMForTokenClassification, LayoutLMv2Processor
+import torch
+from subprocess import run
+
+# install tesseract-ocr and pytesseract
+run("apt install -y tesseract-ocr", shell=True, check=True)
+run("pip install pytesseract", shell=True, check=True)
+
+# helper function to unnormalize bboxes for drawing onto the image
+def unnormalize_box(bbox, width, height):
+    return [
+        width * (bbox[0] / 1000),
+        height * (bbox[1] / 1000),
+        width * (bbox[2] / 1000),
+        height * (bbox[3] / 1000),
+    ]
+
+# set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load model and processor from path
+        self.model = LayoutLMForTokenClassification.from_pretrained(path).to(device)
+        self.processor = LayoutLMv2Processor.from_pretrained(path)
+
+    def __call__(self, data: Dict[str, bytes]) -> Dict[str, List[Any]]:
+        """
+        Args:
+            data (:obj:):
+                includes the deserialized image file as PIL.Image
+        """
+        # process input
+        image = data.pop("inputs", data)
+
+        # process image
+        encoding = self.processor(image, return_tensors="pt")
+
+        # run prediction
+        with torch.inference_mode():
+            outputs = self.model(
+                input_ids=encoding.input_ids.to(device),
+                bbox=encoding.bbox.to(device),
+                attention_mask=encoding.attention_mask.to(device),
+                token_type_ids=encoding.token_type_ids.to(device),
+            )
+            predictions = outputs.logits.softmax(-1)
+
+        # post process output
+        result = []
+        for item, inp_ids, bbox in zip(
+            predictions.squeeze(0).cpu(), encoding.input_ids.squeeze(0).cpu(), encoding.bbox.squeeze(0).cpu()
+        ):
+            label = self.model.config.id2label[int(item.argmax().cpu())]
+            if label == "O":
+                continue
+            score = item.max().item()
+            text = self.processor.tokenizer.decode(inp_ids)
+            bbox = unnormalize_box(bbox.tolist(), image.width, image.height)
+            result.append({"label": label, "score": score, "text": text, "bbox": bbox})
+        return {"predictions": result}
+```
+
+### 2. Send HTTP request using Python
+
+Hugging Face Inference endpoints can directly work with binary data, this means that we can directly send our image from our document to the endpoint. We are going to use `requests` to send our requests. (make your you have it installed `pip install requests`)
+
+```python
+import json
+import requests as r
+import mimetypes
+
+ENDPOINT_URL="" # url of your endpoint
+HF_TOKEN="" # organization token where you deployed your endpoint
+
+def predict(path_to_image:str=None):
+    with open(path_to_image, "rb") as i:
+      b = i.read()
+    headers= {
+        "Authorization": f"Bearer {HF_TOKEN}",
+        "Content-Type": mimetypes.guess_type(path_to_image)[0]
+    }
+    response = r.post(ENDPOINT_URL, headers=headers, data=b)
+    return response.json()
+
+prediction = predict(path_to_image="path_to_your_image.png")
+
+print(prediction)
+# {'predictions': [{'label': 'I-ANSWER', 'score': 0.4823932945728302, 'text': '[CLS]', 'bbox': [0.0, 0.0, 0.0, 0.0]}, {'label': 'B-HEADER', 'score': 0.992474377155304, 'text': 'your', 'bbox': [1712.529, 181.203, 1859.949, 228.88799999999998]},
+```
+
+
+### 3. Draw result on image
+
+To get a better understanding of what the model predicted you can also draw the predictions on the provided image. 
+
+```python
+from PIL import Image, ImageDraw, ImageFont
+
+# draw results on image
+def draw_result(path_to_image,result):
+  image = Image.open(path_to_image)
+  label2color = {
+      "B-HEADER": "blue",
+      "B-QUESTION": "red",
+      "B-ANSWER": "green",
+      "I-HEADER": "blue",
+      "I-QUESTION": "red",
+      "I-ANSWER": "green",
+  }
+
+  # draw predictions over the image
+  draw = ImageDraw.Draw(image)
+  font = ImageFont.load_default()
+  for res in result:
+      draw.rectangle(res["bbox"], outline="black")
+      draw.rectangle(res["bbox"], outline=label2color[res["label"]])
+      draw.text((res["bbox"][0] + 10, res["bbox"][1] - 10), text=res["label"], fill=label2color[res["label"]], font=font)
+  return image
+
+draw_result("path_to_your_image.png", prediction["predictions"])
+```

config.json

@@ -0,0 +1,45 @@
+{
+  "_name_or_path": "microsoft/layoutlm-base-uncased",
+  "architectures": [
+    "LayoutLMForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "B-HEADER",
+    "2": "I-HEADER",
+    "3": "B-QUESTION",
+    "4": "I-QUESTION",
+    "5": "B-ANSWER",
+    "6": "I-ANSWER"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "B-ANSWER": 5,
+    "B-HEADER": 1,
+    "B-QUESTION": 3,
+    "I-ANSWER": 6,
+    "I-HEADER": 2,
+    "I-QUESTION": 4,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_2d_position_embeddings": 1024,
+  "max_position_embeddings": 512,
+  "model_type": "layoutlm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

create_handler.ipynb

@@ -0,0 +1,223 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Setup & Installation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!apt install -y tesseract-ocr\n",
+    "pip install pytesseract"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Create Custom Handler for Inference Endpoints\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting handler.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile handler.py\n",
+    "from typing import Dict, List, Any\n",
+    "from transformers import LayoutLMForTokenClassification, LayoutLMv2Processor\n",
+    "import torch\n",
+    "from subprocess import run\n",
+    "\n",
+    "# install tesseract-ocr and pytesseract\n",
+    "run(\"apt install -y tesseract-ocr\", shell=True, check=True)\n",
+    "run(\"pip install pytesseract\", shell=True, check=True)\n",
+    "\n",
+    "# helper function to unnormalize bboxes for drawing onto the image\n",
+    "def unnormalize_box(bbox, width, height):\n",
+    "    return [\n",
+    "        width * (bbox[0] / 1000),\n",
+    "        height * (bbox[1] / 1000),\n",
+    "        width * (bbox[2] / 1000),\n",
+    "        height * (bbox[3] / 1000),\n",
+    "    ]\n",
+    "\n",
+    "\n",
+    "# set device\n",
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "\n",
+    "\n",
+    "class EndpointHandler:\n",
+    "    def __init__(self, path=\"\"):\n",
+    "        # load model and processor from path\n",
+    "        self.model = LayoutLMForTokenClassification.from_pretrained(\"philschmid/layoutlm-funsd\").to(device)\n",
+    "        self.processor = LayoutLMv2Processor.from_pretrained(\"philschmid/layoutlm-funsd\")\n",
+    "\n",
+    "    def __call__(self, data: Dict[str, bytes]) -> Dict[str, List[Any]]:\n",
+    "        \"\"\"\n",
+    "        Args:\n",
+    "            data (:obj:):\n",
+    "                includes the deserialized image file as PIL.Image\n",
+    "        \"\"\"\n",
+    "        # process input\n",
+    "        image = data.pop(\"inputs\", data)\n",
+    "\n",
+    "        # process image\n",
+    "        encoding = self.processor(image, return_tensors=\"pt\")\n",
+    "\n",
+    "        # run prediction\n",
+    "        with torch.inference_mode():\n",
+    "            outputs = self.model(\n",
+    "                input_ids=encoding.input_ids.to(device),\n",
+    "                bbox=encoding.bbox.to(device),\n",
+    "                attention_mask=encoding.attention_mask.to(device),\n",
+    "                token_type_ids=encoding.token_type_ids.to(device),\n",
+    "            )\n",
+    "            predictions = outputs.logits.softmax(-1)\n",
+    "\n",
+    "        # post process output\n",
+    "        result = []\n",
+    "        for item, inp_ids, bbox in zip(\n",
+    "            predictions.squeeze(0).cpu(), encoding.input_ids.squeeze(0).cpu(), encoding.bbox.squeeze(0).cpu()\n",
+    "        ):\n",
+    "            label = self.model.config.id2label[int(item.argmax().cpu())]\n",
+    "            if label == \"O\":\n",
+    "                continue\n",
+    "            score = item.max().item()\n",
+    "            text = self.processor.tokenizer.decode(inp_ids)\n",
+    "            bbox = unnormalize_box(bbox.tolist(), image.width, image.height)\n",
+    "            result.append({\"label\": label, \"score\": score, \"text\": text, \"bbox\": bbox})\n",
+    "        return {\"predictions\": result}\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "test custom pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from handler import EndpointHandler\n",
+    "\n",
+    "my_handler = EndpointHandler(\".\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import base64\n",
+    "from PIL import Image\n",
+    "from io import BytesIO\n",
+    "import json\n",
+    "\n",
+    "# read image from disk\n",
+    "image = Image.open(\"invoice_example.png\")\n",
+    "request = {\"inputs\":image }\n",
+    "\n",
+    "# test the handler\n",
+    "pred = my_handler(request)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import Image, ImageDraw, ImageFont\n",
+    "\n",
+    "\n",
+    "def draw_result(image,result):\n",
+    "    label2color = {\n",
+    "        \"B-HEADER\": \"blue\",\n",
+    "        \"B-QUESTION\": \"red\",\n",
+    "        \"B-ANSWER\": \"green\",\n",
+    "        \"I-HEADER\": \"blue\",\n",
+    "        \"I-QUESTION\": \"red\",\n",
+    "        \"I-ANSWER\": \"green\",\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "    # draw predictions over the image\n",
+    "    draw = ImageDraw.Draw(image)\n",
+    "    font = ImageFont.load_default()\n",
+    "    for res in result:\n",
+    "        draw.rectangle(res[\"bbox\"], outline=\"black\")\n",
+    "        draw.rectangle(res[\"bbox\"], outline=label2color[res[\"label\"]])\n",
+    "        draw.text((res[\"bbox\"][0] + 10, res[\"bbox\"][1] - 10), text=res[\"label\"], fill=label2color[res[\"label\"]], font=font)\n",
+    "    return image\n",
+    "\n",
+    "draw_result(image,pred[\"predictions\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.13 ('dev': conda)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "f6dd96c16031089903d5a31ec148b80aeb0d39c32affb1a1080393235fbfa2fc"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

handler.py

@@ -0,0 +1,64 @@
+from typing import Dict, List, Any
+from transformers import LayoutLMForTokenClassification, LayoutLMv2Processor
+import torch
+from subprocess import run
+
+# install tesseract-ocr and pytesseract
+run("apt install -y tesseract-ocr", shell=True, check=True)
+run("pip install pytesseract", shell=True, check=True)
+
+# helper function to unnormalize bboxes for drawing onto the image
+def unnormalize_box(bbox, width, height):
+    return [
+        width * (bbox[0] / 1000),
+        height * (bbox[1] / 1000),
+        width * (bbox[2] / 1000),
+        height * (bbox[3] / 1000),
+    ]
+
+
+# set device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load model and processor from path
+        self.model = LayoutLMForTokenClassification.from_pretrained(path).to(device)
+        self.processor = LayoutLMv2Processor.from_pretrained(path)
+
+    def __call__(self, data: Dict[str, bytes]) -> Dict[str, List[Any]]:
+        """
+        Args:
+            data (:obj:):
+                includes the deserialized image file as PIL.Image
+        """
+        # process input
+        image = data.pop("inputs", data)
+
+        # process image
+        encoding = self.processor(image, return_tensors="pt", truncation=True)
+
+        # run prediction
+        with torch.inference_mode():
+            outputs = self.model(
+                input_ids=encoding.input_ids.to(device),
+                bbox=encoding.bbox.to(device),
+                attention_mask=encoding.attention_mask.to(device),
+                token_type_ids=encoding.token_type_ids.to(device),
+            )
+            predictions = outputs.logits.softmax(-1)
+
+        # post process output
+        result = []
+        for item, inp_ids, bbox in zip(
+            predictions.squeeze(0).cpu(), encoding.input_ids.squeeze(0).cpu(), encoding.bbox.squeeze(0).cpu()
+        ):
+            label = self.model.config.id2label[int(item.argmax().cpu())]
+            if label == "O":
+                continue
+            score = item.max().item()
+            text = self.processor.tokenizer.decode(inp_ids)
+            bbox = unnormalize_box(bbox.tolist(), image.width, image.height)
+            result.append({"label": label, "score": score, "text": text, "bbox": bbox})
+        return {"predictions": result}

preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "apply_ocr": true,
+  "do_resize": true,
+  "feature_extractor_type": "LayoutLMv2FeatureExtractor",
+  "ocr_lang": null,
+  "processor_class": "LayoutLMv2Processor",
+  "resample": 2,
+  "size": 224
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f31380262cd4f276be211189196f190c0268e9cece977d500886a4e4c16fc07
+size 450606565

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": null,
+  "apply_ocr": false,
+  "cls_token": "[CLS]",
+  "cls_token_box": [
+    0,
+    0,
+    0,
+    0
+  ],
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "microsoft/layoutlmv2-base-uncased",
+  "never_split": null,
+  "only_label_first_subword": true,
+  "pad_token": "[PAD]",
+  "pad_token_box": [
+    0,
+    0,
+    0,
+    0
+  ],
+  "pad_token_label": -100,
+  "processor_class": "LayoutLMv2Processor",
+  "sep_token": "[SEP]",
+  "sep_token_box": [
+    1000,
+    1000,
+    1000,
+    1000
+  ],
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "LayoutLMv2Tokenizer",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56fc4a68a8102016f0d13df85e3cef173b08bfd50400f2f88c520a325d11676
+size 3375