Initial Commit

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/DocVQA-Sanctum.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,104 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="HtmlUnknownTag" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="myValues">
+        <value>
+          <list size="7">
+            <item index="0" class="java.lang.String" itemvalue="nobr" />
+            <item index="1" class="java.lang.String" itemvalue="noembed" />
+            <item index="2" class="java.lang.String" itemvalue="comment" />
+            <item index="3" class="java.lang.String" itemvalue="noscript" />
+            <item index="4" class="java.lang.String" itemvalue="embed" />
+            <item index="5" class="java.lang.String" itemvalue="script" />
+            <item index="6" class="java.lang.String" itemvalue="style" />
+          </list>
+        </value>
+      </option>
+      <option name="myCustomValuesEnabled" value="true" />
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="67">
+            <item index="0" class="java.lang.String" itemvalue="absl-py" />
+            <item index="1" class="java.lang.String" itemvalue="networkx" />
+            <item index="2" class="java.lang.String" itemvalue="huggingface-hub" />
+            <item index="3" class="java.lang.String" itemvalue="PyYAML" />
+            <item index="4" class="java.lang.String" itemvalue="gast" />
+            <item index="5" class="java.lang.String" itemvalue="MarkupSafe" />
+            <item index="6" class="java.lang.String" itemvalue="numpy" />
+            <item index="7" class="java.lang.String" itemvalue="pyasn1" />
+            <item index="8" class="java.lang.String" itemvalue="requests" />
+            <item index="9" class="java.lang.String" itemvalue="Jinja2" />
+            <item index="10" class="java.lang.String" itemvalue="fsspec" />
+            <item index="11" class="java.lang.String" itemvalue="pyasn1-modules" />
+            <item index="12" class="java.lang.String" itemvalue="safetensors" />
+            <item index="13" class="java.lang.String" itemvalue="certifi" />
+            <item index="14" class="java.lang.String" itemvalue="keras" />
+            <item index="15" class="java.lang.String" itemvalue="urllib3" />
+            <item index="16" class="java.lang.String" itemvalue="itsdangerous" />
+            <item index="17" class="java.lang.String" itemvalue="Markdown" />
+            <item index="18" class="java.lang.String" itemvalue="sympy" />
+            <item index="19" class="java.lang.String" itemvalue="Flask" />
+            <item index="20" class="java.lang.String" itemvalue="blinker" />
+            <item index="21" class="java.lang.String" itemvalue="tokenizers" />
+            <item index="22" class="java.lang.String" itemvalue="libclang" />
+            <item index="23" class="java.lang.String" itemvalue="transformers" />
+            <item index="24" class="java.lang.String" itemvalue="google-auth-oauthlib" />
+            <item index="25" class="java.lang.String" itemvalue="Werkzeug" />
+            <item index="26" class="java.lang.String" itemvalue="h5py" />
+            <item index="27" class="java.lang.String" itemvalue="tensorboard-data-server" />
+            <item index="28" class="java.lang.String" itemvalue="packaging" />
+            <item index="29" class="java.lang.String" itemvalue="torch" />
+            <item index="30" class="java.lang.String" itemvalue="click" />
+            <item index="31" class="java.lang.String" itemvalue="tqdm" />
+            <item index="32" class="java.lang.String" itemvalue="termcolor" />
+            <item index="33" class="java.lang.String" itemvalue="regex" />
+            <item index="34" class="java.lang.String" itemvalue="mpmath" />
+            <item index="35" class="java.lang.String" itemvalue="typing_extensions" />
+            <item index="36" class="java.lang.String" itemvalue="cachetools" />
+            <item index="37" class="java.lang.String" itemvalue="charset-normalizer" />
+            <item index="38" class="java.lang.String" itemvalue="grpcio" />
+            <item index="39" class="java.lang.String" itemvalue="gTTS" />
+            <item index="40" class="java.lang.String" itemvalue="google-auth" />
+            <item index="41" class="java.lang.String" itemvalue="idna" />
+            <item index="42" class="java.lang.String" itemvalue="referencing" />
+            <item index="43" class="java.lang.String" itemvalue="tzdata" />
+            <item index="44" class="java.lang.String" itemvalue="kiwisolver" />
+            <item index="45" class="java.lang.String" itemvalue="rich" />
+            <item index="46" class="java.lang.String" itemvalue="cycler" />
+            <item index="47" class="java.lang.String" itemvalue="sniffio" />
+            <item index="48" class="java.lang.String" itemvalue="markdown-it-py" />
+            <item index="49" class="java.lang.String" itemvalue="attrs" />
+            <item index="50" class="java.lang.String" itemvalue="contourpy" />
+            <item index="51" class="java.lang.String" itemvalue="jsonschema-specifications" />
+            <item index="52" class="java.lang.String" itemvalue="pandas" />
+            <item index="53" class="java.lang.String" itemvalue="exceptiongroup" />
+            <item index="54" class="java.lang.String" itemvalue="fonttools" />
+            <item index="55" class="java.lang.String" itemvalue="rpds-py" />
+            <item index="56" class="java.lang.String" itemvalue="toolz" />
+            <item index="57" class="java.lang.String" itemvalue="Pygments" />
+            <item index="58" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="59" class="java.lang.String" itemvalue="anyio" />
+            <item index="60" class="java.lang.String" itemvalue="pillow" />
+            <item index="61" class="java.lang.String" itemvalue="pytz" />
+            <item index="62" class="java.lang.String" itemvalue="uvicorn" />
+            <item index="63" class="java.lang.String" itemvalue="pyparsing" />
+            <item index="64" class="java.lang.String" itemvalue="jsonschema" />
+            <item index="65" class="java.lang.String" itemvalue="pydantic_core" />
+            <item index="66" class="java.lang.String" itemvalue="gradio_client" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N801" />
+          <option value="N813" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (docvqa_venv)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/DocVQA-Sanctum.iml" filepath="$PROJECT_DIR$/.idea/DocVQA-Sanctum.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

app.py

@@ -0,0 +1,47 @@
+import gradio as gr
+import warnings
+import os
+import pix2struct, layoutlm, donut
+warnings.filterwarnings('ignore')
+
+
+def process_image_and_generate_output(image, model_selection, question):
+    result = ''
+    if image is None:
+        return "Please select an image", None
+
+    if model_selection == "LayoutLM":
+        result = layoutlm.get_result(image, question)
+        return result
+    if model_selection == 'Pix2Struct':
+        result = pix2struct.get_result(image, question)
+        return result
+    if model_selection == 'Donut':
+        result = donut.get_result(image, question)
+        return result
+
+    return result
+
+
+sample_images = [
+    [os.path.join(os.path.dirname(__file__), "images/1.png"), "LayoutLM", "What is the NIC Code?"],
+    [os.path.join(os.path.dirname(__file__), "images/1.png"), "Pix2Struct", "What is the NIC Code?"],
+    [os.path.join(os.path.dirname(__file__), "images/1.png"), "Donut", "What is the NIC Code?"]
+]
+
+# Create a dropdown to select sample image
+image_input = gr.Image(label="Upload Image", type='filepath')
+
+# Create a dropdown to choose the model
+model_selection_input = gr.Radio(["LayoutLM", "Pix2Struct", "Donut"],
+                                 label="Choose Model")
+question_input = gr.Text(label="Question")
+
+iface = gr.Interface(fn=process_image_and_generate_output,
+                     inputs=[image_input, model_selection_input, question_input],
+                     outputs=gr.Text(label="Result"),
+                     allow_flagging='never',
+                     examples=sample_images,
+                     title="DocVQA Sanctum")
+
+iface.launch()

donut.py

@@ -0,0 +1,36 @@
+import re
+from PIL import Image
+from transformers import DonutProcessor, VisionEncoderDecoderModel
+
+
+def get_result(image_path, question):
+    processor = DonutProcessor.from_pretrained("naver-clova-ix/donut-base-finetuned-docvqa")
+    model = VisionEncoderDecoderModel.from_pretrained("naver-clova-ix/donut-base-finetuned-docvqa")
+
+    # load document image from the DocVQA dataset
+    image = Image.open(image_path).convert('RGB')
+
+    # prepare decoder inputs
+    task_prompt = "<s_docvqa><s_question>{user_input}</s_question><s_answer>"
+    prompt = task_prompt.replace("{user_input}", question)
+    decoder_input_ids = processor.tokenizer(prompt, add_special_tokens=False, return_tensors="pt").input_ids
+
+    pixel_values = processor(image, return_tensors="pt").pixel_values
+
+    outputs = model.generate(
+        pixel_values,
+        decoder_input_ids=decoder_input_ids,
+        max_length=model.decoder.config.max_position_embeddings,
+        pad_token_id=processor.tokenizer.pad_token_id,
+        eos_token_id=processor.tokenizer.eos_token_id,
+        use_cache=True,
+        bad_words_ids=[[processor.tokenizer.unk_token_id]],
+        return_dict_in_generate=True,
+    )
+
+    sequence = processor.batch_decode(outputs.sequences)[0]
+    sequence = sequence.replace(processor.tokenizer.eos_token, "").replace(processor.tokenizer.pad_token, "")
+    sequence = re.sub(r"<.*?>", "", sequence, count=1).strip()  # remove first task start token
+    print(processor.token2json(sequence))
+
+    return processor.token2json(sequence)['answer']

layoutlm.py

@@ -0,0 +1,12 @@
+from transformers import pipeline
+
+
+def get_result(image_path, question):
+    nlp = pipeline(
+        "document-question-answering",
+        model="impira/layoutlm-document-qa",
+    )
+
+    result = nlp(image_path, question)
+
+    return result[0]['answer']

packages.txt

@@ -0,0 +1,4 @@
+poppler-utils
+tesseract-ocr
+chromium
+chromium-driver

pix2struct.py

@@ -0,0 +1,15 @@
+from PIL import Image
+from transformers import Pix2StructForConditionalGeneration as psg
+from transformers import Pix2StructProcessor as psp
+
+
+def get_result(image_path, question):
+    model = psg.from_pretrained("google/pix2struct-docvqa-large")
+    processor = psp.from_pretrained("google/pix2struct-docvqa-large")
+
+    image = Image.open(image_path).convert("RGB")
+    inputs = processor(images=image, text=question, return_tensors="pt")
+    predictions = model.generate(**inputs, max_new_tokens=256)
+    predicted_answer = processor.batch_decode(predictions, skip_special_tokens=True)
+
+    return predicted_answer