first commit

app.py

@@ -1,7 +1,218 @@
+import os
+import subprocess
+from PIL import Image
+import io
 import gradio as gr
+from transformers import AutoProcessor, TextIteratorStreamer
+from transformers import Idefics2ForConditionalGeneration
+import torch
+from peft import LoraConfig
+from transformers import AutoProcessor, BitsAndBytesConfig, IdeficsForVisionText2Text
 
-def greet(name):
-    return "Hello " + name + "!!"
+# Project description
+description = """
+# Kalbe Farma - Visual Question Answering (VQA) for Medical Imaging
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+## Overview
+The project addresses the challenge of accurate and efficient medical imaging analysis in healthcare, aiming to reduce human error and workload for radiologists. The proposed solution involves developing advanced AI models for Visual Question Answering (VQA) to assist healthcare professionals in analyzing medical images quickly and accurately. These models will be integrated into a user-friendly web application, providing a practical tool for real-world healthcare settings.
+
+## Dataset
+The model is trained using the [Hugging face](https://huggingface.co/datasets/flaviagiammarino/vqa-rad/viewer).
+
+Reference: [ScienceDirect](https://www.sciencedirect.com/science/article/abs/pii/S0933365723001252)
+
+## Model Architecture
+
+![Model Architecture](img/Model-Architecture.png)
+
+Reference: [ScienceDirect](https://www.sciencedirect.com/science/article/abs/pii/S0933365723001252)
+
+## Demo
+Please select the example below or upload 4 pairs of mammography exam results.
+"""
+
+DEVICE = torch.device("cuda")
+
+USE_LORA = False
+USE_QLORA = True
+
+if USE_QLORA or USE_LORA:
+    lora_config = LoraConfig(
+        r=8,
+        lora_alpha=8,
+        lora_dropout=0.1,
+        target_modules='.*(text_model|modality_projection|perceiver_resampler).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$',
+        use_dora=False if USE_QLORA else True,
+        init_lora_weights="gaussian"
+    )
+    if USE_QLORA:
+        bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.float16
+        )
+    # Model
+    model = Idefics2ForConditionalGeneration.from_pretrained(
+        "jihadzakki/idefics2-8b-vqarad-delta",
+        torch_dtype=torch.float16,
+        quantization_config=bnb_config
+    )
+
+
+processor = AutoProcessor.from_pretrained(
+    "HuggingFaceM4/idefics2-8b",
+)
+
+def format_answer(image, question, history):
+    try:
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image"},
+                    {"type": "text", "text": question}
+                ]
+            }
+        ]
+
+        text = processor.apply_chat_template(messages, add_generation_prompt=True)
+        inputs = processor(text=[text.strip()], images=[image], return_tensors="pt", padding=True)
+        inputs = {key: value.to(DEVICE) for key, value in inputs.items()}
+        generated_ids = model.generate(**inputs, max_new_tokens=64)
+        generated_texts = processor.batch_decode(generated_ids[:, inputs["input_ids"].size(1):], skip_special_tokens=True)[0]
+
+        history.append((image, f"Question: {question} | Answer: {generated_texts}"))
+
+        # Store the predicted answer in a variable before deleting intermediate variables
+        predicted_answer = f"Predicted Answer: {generated_texts}"
+
+        # Clear the cache and delete unnecessary variables
+        del inputs
+        del generated_ids
+        del generated_texts
+        torch.cuda.empty_cache()
+
+        return predicted_answer, history
+    except Exception as e:
+        # Clear the cache in case of an error
+        torch.cuda.empty_cache()
+        return f"Error: {str(e)}", history
+
+def clear_history():
+    return "", []
+
+def undo_last(history):
+    if history:
+        history.pop()
+    return "", history
+
+def retry_last(image, question, history):
+    if history:
+        last_image, last_entry = history[-1]
+        return format_answer(last_image, question, history[:-1])
+    return "No previous analysis to retry.", history
+
+def switch_theme(mode):
+    if mode == "Light Mode":
+        return gr.themes.Default()
+    else:
+        return gr.themes.Soft(primary_hue=gr.themes.colors.blue, secondary_hue=gr.themes.colors.orange)
+
+def save_feedback(feedback):
+    return "Thank you for your feedback!"
+
+def display_history(history):
+    log_entries = []
+    for img, text in history:
+        log_entries.append((img, text))
+    return log_entries
+
+# Build the Visual QA application using Gradio with improvements
+with gr.Blocks(
+    theme=gr.themes.Soft(
+        font=[gr.themes.GoogleFont("Inconsolata"), "Arial", "sans-serif"],
+        primary_hue=gr.themes.colors.blue,
+        secondary_hue=gr.themes.colors.red,
+    )
+) as VisualQAApp:
+    gr.Markdown(description, elem_classes="title")  # Display the project description
+
+    gr.Markdown("## Demo")
+
+    with gr.Row():
+        with gr.Column():
+            image_input = gr.Image(label="Upload image", type="pil")
+            question_input = gr.Textbox(show_label=False, placeholder="Enter your question here...")
+            submit_button = gr.Button("Submit", variant="primary")
+
+        with gr.Column():
+            answer_output = gr.Textbox(label="Result Prediction")
+
+    history_state = gr.State([])  # Initialize the history state
+
+    submit_button.click(
+        format_answer,
+        inputs=[image_input, question_input, history_state],
+        outputs=[answer_output, history_state],
+        show_progress=True
+    )
+
+    with gr.Row():
+        retry_button = gr.Button("Retry")
+        undo_button = gr.Button("Undo")
+        clear_button = gr.Button("Clear")
+
+        retry_button.click(
+            retry_last,
+            inputs=[image_input, question_input, history_state],
+            outputs=[answer_output, history_state]
+        )
+
+        undo_button.click(
+            undo_last,
+            inputs=[history_state],
+            outputs=[answer_output, history_state]
+        )
+
+        clear_button.click(
+            clear_history,
+            inputs=[],
+            outputs=[answer_output, history_state]
+        )
+
+    with gr.Row():
+        history_gallery = gr.Gallery(label="History Log", elem_id="history_log")
+        submit_button.click(
+            display_history,
+            inputs=[history_state],
+            outputs=[history_gallery]
+        )
+
+    with gr.Accordion("Help", open=False):
+        gr.Markdown("**Upload image**: Select the chest X-ray image you want to analyze.")
+        gr.Markdown("**Enter your question**: Type the question you have about the image, such as 'Is there any sign of pneumonia?'")
+        gr.Markdown("**Submit**: Click the submit button to get the prediction from the model.")
+
+    with gr.Accordion("User Preferences", open=False):
+        gr.Markdown("**Mode**: Choose between light and dark mode for your comfort.")
+        mode_selector = gr.Radio(choices=["Light Mode", "Dark Mode"], label="Select Mode")
+        apply_theme_button = gr.Button("Apply Theme")
+
+        apply_theme_button.click(
+            switch_theme,
+            inputs=[mode_selector],
+            outputs=[],
+        )
+
+    with gr.Accordion("Feedback", open=False):
+        gr.Markdown("**We value your feedback!** Please provide any feedback you have about this application.")
+        feedback_input = gr.Textbox(label="Feedback", lines=4)
+        submit_feedback_button = gr.Button("Submit Feedback")
+
+        submit_feedback_button.click(
+            save_feedback,
+            inputs=[feedback_input],
+            outputs=[feedback_input]
+        )
+
+VisualQAApp.launch(share=True, debug=True)

requirements.txt

@@ -0,0 +1,10 @@
+transformers
+gradio
+plotly
+torch
+tensorflow
+tf-keras
+peft
+bitsandbytes
+accelerate
+pillow