Upload 7 files

app.py

@@ -0,0 +1,181 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import gradio as gr
+from PIL import Image
+from langchain_core.messages import HumanMessage, AIMessage
+
+from graph import build_graph
+from state import WorkflowState
+
+diagnosis_graph = None
+reply_graph = None
+initialization_error = None
+
+try:
+    diagnosis_graph, reply_graph = build_graph() 
+    print("Gradio app and graphs initialized successfully.")
+except Exception as e:
+    initialization_error = f"CRITICAL ERROR: Could not build graphs. {e}. Check API keys."
+    print(initialization_error)
+
+
+def convert_history_to_langchain(chat_history):
+    """Converts Gradio history to Langchain message list."""
+    messages = []
+    for user_msg, ai_msg in chat_history:
+        if user_msg is not None:
+            if isinstance(user_msg, (dict, tuple)):
+                messages.append(HumanMessage(content="User uploaded an image."))
+            else:
+                 messages.append(HumanMessage(content=user_msg))
+        if ai_msg is not None:
+            messages.append(AIMessage(content=ai_msg))
+    return messages
+
+def reset_state_on_start():
+    """Fresh empty state for every new diagnosis."""
+    return WorkflowState(
+        image=None,
+        chat_history=[],
+        disease_prediction="",
+        symptoms_to_check=[],
+        symptoms_confirmed=[],
+        current_symptom_index=0,
+        treatment_info="",
+        final_diagnosis=""
+    )
+
+
+def chat_fn(message: str, chat_history: list, agent_state: dict, img_upload: Image):
+    """
+    Handles user input and manages the agent's workflow state.
+    """
+    if initialization_error:
+        chat_history.append((message, initialization_error))
+        yield chat_history, {}, gr.update(value=None, interactive=True), gr.update(value="", interactive=True)
+        return
+
+    if not agent_state or agent_state.get("final_diagnosis"):
+        current_state = reset_state_on_start()
+    else:
+        current_state = agent_state
+
+    chat_history = chat_history or []
+    
+
+    is_new_diagnosis = False
+    if img_upload and (message.lower().strip() == "start" or message == ""):
+        print("--- Running NEW diagnosis flow ---")
+        is_new_diagnosis = True
+        current_state = reset_state_on_start() 
+        current_state["image"] = img_upload
+        graph_to_run = diagnosis_graph
+        chat_history.append([(img_upload,), None]) 
+        
+    elif current_state.get("symptoms_to_check") and not current_state.get("final_diagnosis"):
+        print("--- Running REPLY symptom loop flow ---")
+        graph_to_run = reply_graph
+        chat_history.append([message, None])
+
+
+    else:
+
+        if message: 
+            chat_history.append([message, None])
+        chat_history[-1][1] = "Hello! Please upload an image, then click 'Start Diagnosis'."
+
+        yield chat_history, agent_state, gr.update(value=img_upload, interactive=True), gr.update(value="", interactive=True)
+        return
+
+    current_state["chat_history"] = convert_history_to_langchain(chat_history)
+    
+    try:
+        final_state = {}
+        for step in graph_to_run.stream(current_state, {"recursion_limit": 100}):
+
+            final_state = list(step.values())[0]
+
+        ai_response = final_state['chat_history'][-1].content
+        chat_history[-1][1] = ai_response 
+
+        if final_state.get("final_diagnosis"):
+            print("--- Agent Flow ENDED ---")
+            yield chat_history, {}, gr.update(value=None, interactive=True), gr.update(value="", interactive=True)
+        else:
+            yield chat_history, final_state, gr.update(value=img_upload, interactive=False), gr.update(value="", interactive=True)
+
+    except Exception as e:
+        print(f"--- Graph Runtime Error --- \n{e}")
+        error_msg = f"A runtime error occurred: {e}. Please check the console."
+        chat_history[-1][1] = error_msg
+        yield chat_history, {}, gr.update(value=None, interactive=True), gr.update(value="", interactive=True)
+
+def clear_all():
+    """Clears chat, state, and image."""
+    return [], {}, None, ""
+
+with gr.Blocks(theme=gr.themes.Soft(), title="Agentic Skin AI") as demo:
+    gr.Markdown("# 🩺 Multimodal Agentic Skin Disease AI")
+    gr.Markdown(
+        "**Disclaimer:** This is a demo project and NOT a medical device. "
+        "Consult a real doctor for any medical concerns."
+    )
+    agent_state = gr.State({})
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("### 1. Image Input")
+            img_upload = gr.Image(type="pil", label="Upload Skin Image", interactive=True)
+            
+            btn_start = gr.Button("Start Diagnosis", variant="primary")
+            btn_clear = gr.Button("Clear All & Start New")
+            
+            gr.Markdown(
+                "**Instructions:**\n"
+                "1. Upload an image.\n"
+                "2. Click **Start Diagnosis**.\n"
+                "3. Answer the agent's questions in the textbox."
+            )
+
+        with gr.Column(scale=2):
+            gr.Markdown("### 2. Agent Conversation")
+            chatbot = gr.Chatbot(label="Agent Conversation", height=500, bubble_full_width=False, avatar_images=None)
+            txt_msg = gr.Textbox(
+                label="Your message (Yes / No / etc.)",
+                placeholder="Answer the agent's questions here...",
+                interactive=True
+            )
+            
+    txt_msg.submit(
+        fn=chat_fn,
+        inputs=[txt_msg, chatbot, agent_state, img_upload],
+        outputs=[chatbot, agent_state, img_upload, txt_msg]
+    )
+
+    btn_start.click(
+        fn=chat_fn,
+        inputs=[gr.Textbox(value="start", visible=False), chatbot, agent_state, img_upload], 
+        outputs=[chatbot, agent_state, img_upload, txt_msg]
+    )
+    
+    btn_clear.click(
+        fn=clear_all,
+        inputs=None,
+        outputs=[chatbot, agent_state, img_upload, txt_msg]
+    )
+    
+    img_upload.upload(
+        fn=lambda: ([], {}, "Click 'Start Diagnosis' to begin."),
+        inputs=None,
+        outputs=[chatbot, agent_state, txt_msg]
+    )
+
+if __name__ == "__main__":
+    if initialization_error:
+        print("\n\n*** CANNOT LAUNCH APP: Agent failed to initialize. ***")
+        print(f"*** ERROR: {initialization_error} ***")
+    else:
+        demo.launch(debug=True)

graph.py

@@ -0,0 +1,78 @@
+from langgraph.graph import StateGraph, END
+from state import WorkflowState
+from nodes import (
+    node_analyze_image,
+    node_fetch_symptoms,
+    node_ask_symptom_question,
+    node_process_user_response,
+    node_generate_final_response,
+    router_check_image_analysis,
+    router_should_ask_symptoms,
+    router_should_continue_asking
+)
+
+def build_graph():
+    """
+    Builds and compiles the two agentic workflows:
+    1. The main diagnosis graph.
+    2. The reply-handling graph.
+    """
+    
+    workflow = StateGraph(WorkflowState)
+
+    workflow.add_node("analyze_image", node_analyze_image)
+    workflow.add_node("fetch_symptoms", node_fetch_symptoms)
+    workflow.add_node("ask_symptom_question", node_ask_symptom_question)
+    workflow.add_node("generate_final_response", node_generate_final_response)
+
+    workflow.set_entry_point("analyze_image")
+    
+    workflow.add_conditional_edges(
+        "analyze_image",
+        router_check_image_analysis,
+        {
+            "fetch_symptoms": "fetch_symptoms",
+            "end_error": END  
+        }
+    )
+
+    workflow.add_conditional_edges(
+        "fetch_symptoms",
+        router_should_ask_symptoms,
+        {
+            "ask_symptom_question": "ask_symptom_question",
+            "generate_final_response": "generate_final_response"
+        }
+    )
+    
+    workflow.add_edge("ask_symptom_question", END)
+    workflow.add_edge("generate_final_response", END)
+    
+    diagnosis_graph = workflow.compile()
+    
+    
+    reply_workflow = StateGraph(WorkflowState)
+    
+    reply_workflow.add_node("process_user_response", node_process_user_response)
+    reply_workflow.add_node("ask_symptom_question", node_ask_symptom_question)
+    reply_workflow.add_node("generate_final_response", node_generate_final_response)
+
+    reply_workflow.set_entry_point("process_user_response")
+    
+    reply_workflow.add_conditional_edges(
+        "process_user_response",
+        router_should_continue_asking,
+        {
+            "ask_symptom_question": "ask_symptom_question",
+            "generate_final_response": "generate_final_response"
+        }
+    )
+    
+    reply_workflow.add_edge("ask_symptom_question", END)
+    reply_workflow.add_edge("generate_final_response", END)
+    
+    reply_graph = reply_workflow.compile()
+    
+    print("--- LangGraph Compiled ---")
+    
+    return diagnosis_graph, reply_graph

llms.py

@@ -0,0 +1,76 @@
+import os
+from langchain_huggingface import HuggingFaceEndpoint
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import StrOutputParser, JsonOutputParser
+from langchain_core.runnables import RunnableLambda
+
+LLM_REPO_ID = os.environ.get("LLM_REPO_ID", "Jyo-K/Fine-Tuned-Qwen2.5_1B")
+HF_API_KEY = os.environ.get("HF_API_KEY")
+
+_llm_instance = None
+
+def get_llm():
+    """Lazily initializes and returns the LLM instance."""
+    global _llm_instance
+    if _llm_instance is None:
+        if not HF_API_KEY:
+            raise ValueError("HF_TOKEN environment variable not set. Cannot initialize LLM.")
+        _llm_instance = HuggingFaceEndpoint(
+            #model= "Jyo-K/Fine-Tuned-Qwen2.5_1B",
+            repo_id=LLM_REPO_ID,
+            huggingfacehub_api_token=HF_API_KEY,
+            temperature=0.1,
+            max_new_tokens=256,
+            top_k=50,
+            top_p=0.95
+        )
+        print("--- LLM Initialized ---")
+    return _llm_instance
+
+classifier_prompt = ChatPromptTemplate.from_messages([
+    ("system", (
+        "You are a helpful classification assistant. "
+        "Your task is to classify the user's last response as 'yes', 'no', or 'unclear' "
+        "based on their message. "
+        "User's previous message: '{last_human_message}'"
+        "\nRespond ONLY with a single JSON object in the format: "
+        "{{\"classification\": \"yes\"}} or {{\"classification\": \"no\"}} or {{\"classification\": \"unclear\"}}"
+    ))
+])
+
+
+symptom_classifier_chain = classifier_prompt | get_llm() | JsonOutputParser()
+
+question_prompt = ChatPromptTemplate.from_messages([
+    ("system", (
+        "You are a friendly medical assistant bot. Ask the user if they are experiencing the "
+        "following symptom. Be clear and concise. Do not add any extra greeting or sign-off. "
+        "Symptom: '{symptom}'"
+        "\nExample: Are you experiencing any itchiness or a rash?"
+    ))
+])
+
+question_generation_chain = question_prompt | get_llm() | StrOutputParser()
+
+
+summary_prompt = ChatPromptTemplate.from_messages([
+    ("system", (
+        "You are a helpful medical assistant providing a summary. "
+        "Based on the initial image analysis and confirmed symptoms, generate a summary. "
+        "DO NOT provide a definitive diagnosis. "
+        "Structure your response clearly: "
+        "1. Start by stating the potential condition identified from the image."
+        "2. List the symptoms the user confirmed."
+        "3. Provide the general treatment information found for this condition."
+        "4. **ALWAYS** include the provided disclaimer at the very end."
+        "\n---"
+        "Initial Image Prediction: {disease}"
+        "Confirmed Symptoms: {symptoms}"
+        "Potential Treatment Information: {treatment}"
+        "Disclaimer: {disclaimer}"
+        "\n---"
+        "Generate your summary now."
+    ))
+])
+
+summary_generation_chain = summary_prompt | get_llm() | StrOutputParser()

merge.py

@@ -0,0 +1,133 @@
+"""
+LoRA Adapter Merge Script for M1 Mac
+Merges LoRA adapters with the base Qwen2.5-1.5B model
+"""
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+import os
+import argparse
+
+def merge_lora_weights(
+    base_model_name: str = "unsloth/Qwen2.5-1.5B", 
+    lora_adapter_path: str = "fine_tuned_model", #fine_tuned_model
+    output_path: str = "merged_model",
+    device: str = "mps"  # Use Metal Performance Shaders for M1
+):
+    """
+    Merge LoRA adapters with base model
+    
+    Args:
+        base_model_name: HuggingFace model name or path to base model
+        lora_adapter_path: Path to your LoRA adapter files
+        output_path: Where to save the merged model
+        device: Device to use ('mps' for M1/M2 Mac, 'cpu' for compatibility)
+    """
+    
+    print(f"🚀 Starting LoRA merge process...")
+    print(f"📦 Base model: {base_model_name}")
+    print(f"🔧 LoRA adapters: {lora_adapter_path}")
+    print(f"💾 Output path: {output_path}")
+    
+    # Check if MPS is available (M1/M2 Mac)
+    if device == "mps" and not torch.backends.mps.is_available():
+        print("⚠️  MPS not available, falling back to CPU")
+        device = "cpu"
+    
+    try:
+        # Step 1: Load the base model
+        print("\n📥 Loading base model...")
+        base_model = AutoModelForCausalLM.from_pretrained(
+            base_model_name,
+            torch_dtype=torch.float16 if device == "mps" else torch.float32,
+            device_map={"": device},
+            low_cpu_mem_usage=True
+        )
+        print("✅ Base model loaded successfully")
+        
+        # Step 2: Load the tokenizer
+        print("\n📥 Loading tokenizer...")
+        tokenizer = AutoTokenizer.from_pretrained(lora_adapter_path)
+        print("✅ Tokenizer loaded successfully")
+        
+        # Step 3: Load LoRA adapters
+        print("\n📥 Loading LoRA adapters...")
+        model_with_lora = PeftModel.from_pretrained(
+            base_model,
+            lora_adapter_path,
+            device_map={"": device}
+        )
+        print("✅ LoRA adapters loaded successfully")
+        
+        # Step 4: Merge weights
+        print("\n🔄 Merging LoRA weights into base model...")
+        merged_model = model_with_lora.merge_and_unload()
+        print("✅ Weights merged successfully")
+        
+        # Step 5: Save the merged model
+        print(f"\n💾 Saving merged model to {output_path}...")
+        os.makedirs(output_path, exist_ok=True)
+        
+        merged_model.save_pretrained(
+            output_path,
+            safe_serialization=True,
+            max_shard_size="2GB"
+        )
+        tokenizer.save_pretrained(output_path)
+        
+        print("✅ Merged model saved successfully!")
+        print(f"\n🎉 Complete! Your merged model is ready at: {output_path}")
+        print(f"📊 Model size: ~3GB")
+        
+        # Optional: Print model info
+        print("\n📋 Model Information:")
+        print(f"   - Architecture: {merged_model.config.architectures}")
+        print(f"   - Parameters: {sum(p.numel() for p in merged_model.parameters()):,}")
+        print(f"   - Vocab size: {merged_model.config.vocab_size}")
+        
+        return merged_model, tokenizer
+        
+    except Exception as e:
+        print(f"\n❌ Error during merge process: {str(e)}")
+        raise
+
+def main():
+    parser = argparse.ArgumentParser(description="Merge LoRA adapters with base model")
+    parser.add_argument(
+        "--base-model",
+        type=str,
+        default="unsloth/Qwen2.5-1.5B",
+        help="Base model name or path"
+    )
+    parser.add_argument(
+        "--lora-path",
+        type=str,
+        default="./fine_tuned_model",
+        help="Path to LoRA adapter files"
+    )
+    parser.add_argument(
+        "--output-path",
+        type=str,
+        default="./merged_model",
+        help="Output path for merged model"
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="mps",
+        choices=["mps", "cpu"],
+        help="Device to use (mps for M1/M2, cpu for compatibility)"
+    )
+    
+    args = parser.parse_args()
+    
+    merge_lora_weights(
+        base_model_name=args.base_model,
+        lora_adapter_path=args.lora_path,
+        output_path=args.output_path,
+        device=args.device
+    )
+
+if __name__ == "__main__":
+    main()

nodes.py

@@ -0,0 +1,144 @@
+from langchain_core.messages import AIMessage, HumanMessage
+from state import WorkflowState
+from tools import tool_analyze_skin_image, tool_fetch_disease_info
+from llms import (
+    symptom_classifier_chain, 
+    question_generation_chain, 
+    summary_generation_chain
+)
+
+
+def node_analyze_image(state: WorkflowState) -> WorkflowState:
+    """Analyzes the user's image and updates the state."""
+    print("--- Node: Analyzing Image ---")
+    image = state.get("image")
+    if not image:
+        state['chat_history'].append(AIMessage(
+            content="Please upload an image first."
+        ))
+        return state
+
+    prediction_result = tool_analyze_skin_image.invoke({"image": image})
+    
+    if "Error:" in prediction_result:
+        state['chat_history'].append(AIMessage(content=prediction_result))
+        state['final_diagnosis'] = "Error" 
+        return state
+    
+    state['disease_prediction'] = prediction_result
+    return state
+
+def node_fetch_symptoms(state: WorkflowState) -> WorkflowState:
+    """Fetches symptoms for the predicted disease."""
+    print(f"--- Node: Fetching Symptoms for {state['disease_prediction']} ---")
+    disease = state['disease_prediction']
+    
+    info = tool_fetch_disease_info.invoke({"disease_name": disease})
+    
+    if "error" in info:
+        state['chat_history'].append(AIMessage(content=info['error']))
+        state['final_diagnosis'] = "Error" 
+        return state
+    
+    state['symptoms_to_check'] = info.get("symptoms", [])
+    state['treatment_info'] = info.get("treatment", "No treatment info available.")
+    state['current_symptom_index'] = 0
+    state['symptoms_confirmed'] = []
+    
+    if not state['symptoms_to_check']:
+        print("No symptoms found to check. Proceeding to final response.")
+    
+    return state
+
+def node_ask_symptom_question(state: WorkflowState) -> WorkflowState:
+    """Asks the user the next symptom question."""
+    print(f"--- Node: Asking Symptom Question {state['current_symptom_index']} ---")
+    symptoms = state['symptoms_to_check']
+    index = state['current_symptom_index']
+    
+    symptom = symptoms[index]
+    
+    question = question_generation_chain.invoke({"symptom": symptom})
+    
+    state['chat_history'].append(AIMessage(content=question))
+    state['current_symptom_index'] = index + 1
+    return state
+
+def node_process_user_response(state: WorkflowState) -> WorkflowState:
+    """Processes the user's 'yes' or 'no' response to a symptom question."""
+    print("--- Node: Processing User Response ---")
+    last_human_message = state['chat_history'][-1].content
+    
+    index = state['current_symptom_index']
+    last_asked_symptom = state['symptoms_to_check'][index - 1]
+    
+    try:
+        classification = symptom_classifier_chain.invoke(
+            {"last_human_message": last_human_message}
+        )
+        
+        if classification.get("classification") == "yes":
+            print(f"User confirmed symptom: {last_asked_symptom}")
+            state['symptoms_confirmed'].append(last_asked_symptom)
+        else:
+            print(f"User denied symptom: {last_asked_symptom}")
+            
+    except Exception as e:
+        print(f"Error classifying user response: {e}. Assuming 'unclear'.")
+    
+    return state
+    
+def node_generate_final_response(state: WorkflowState) -> WorkflowState:
+    """Generates the final summary and disclaimer for the user."""
+    print("--- Node: Generating Final Response ---")
+    
+    disclaimer = (
+        "\n\n**DISCLAIMER:**\n"
+        "I am just a dumb agent, not a medical professional. "
+        "This is a side project for learning purposes. "
+        "Please **DO NOT** take this information for face value. "
+        "Consult a real doctor or dermatologist for any medical concerns."
+    )
+    
+    summary = summary_generation_chain.invoke({
+        "disease": state['disease_prediction'],
+        "symptoms": ", ".join(state['symptoms_confirmed']) or "None confirmed",
+        "treatment": state['treatment_info'],
+        "disclaimer": disclaimer
+    })
+    
+    state['chat_history'].append(AIMessage(content=summary))
+    state['final_diagnosis'] = "Complete" 
+    return state
+
+
+def router_should_ask_symptoms(state: WorkflowState) -> str:
+    """
+    Checks if there are symptoms to ask about.
+    If yes -> ask_symptom_question
+    If no -> generate_final_response
+    """
+    if state.get("symptoms_to_check"):
+        return "ask_symptom_question"
+    else:
+        return "generate_final_response"
+
+def router_should_continue_asking(state: WorkflowState) -> str:
+    """
+    Checks if we have more symptoms to ask about after a user's response.
+    If yes -> ask_symptom_question
+    If no -> generate_final_response
+    """
+    if state['current_symptom_index'] < len(state['symptoms_to_check']):
+        return "ask_symptom_question"
+    else:
+        return "generate_final_response"
+
+def router_check_image_analysis(state: WorkflowState) -> str:
+    """
+    Checks if the image analysis was successful.
+    """
+    if state.get("final_diagnosis") == "Error":
+        return "end_error" 
+    else:
+        return "fetch_symptoms"

requirements.txt

@@ -0,0 +1,17 @@
+pydantic
+langchain-pinecone
+langchain
+langchain-community
+ipykernel
+langchain-core
+langchain-text-splitters
+langgraph
+sentence-transformers
+python-dotenv
+gradio
+langchain-huggingface
+pinecone-client
+requests
+pillow
+pypdf
+pinecone-client

state.py

@@ -0,0 +1,20 @@
+from typing import TypedDict, List, Optional
+from langchain_core.messages import BaseMessage
+from PIL.Image import Image
+
+class WorkflowState(TypedDict):
+    """
+    Represents the state of our agent's workflow.
+    This dictionary is passed between nodes, allowing them to share information.
+    """
+    image: Optional[Image]
+    chat_history: List[BaseMessage]
+    
+    disease_prediction: str
+    symptoms_to_check: List[str]
+    treatment_info: str
+    
+    symptoms_confirmed: List[str]
+    current_symptom_index: int
+    
+    final_diagnosis: str