app.py

from flask import Flask, request, jsonify
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
from qwen_vl_utils import process_vision_info
import torch
import pandas as pd
import pytesseract
import cv2
import pymssql

app = Flask(__name__)

# Initialize model and processor
model = Qwen2VLForConditionalGeneration.from_pretrained("Qwen/Qwen2-VL-2B-Instruct-AWQ", torch_dtype="auto")
if torch.cuda.is_available():
    model.to("cuda")

processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct-AWQ")
pytesseract.pytesseract_cmd = r'/usr/bin/tesseract'

# Function to identify category based on keywords
def identify_category(text):
    text = text.lower()
    if any(keyword in text for keyword in ["food", "meal", "restaurant", "cafe", "coffee", "drink"]):
        return "Food"
    elif any(keyword in text for keyword in ["travel", "flight", "bus", "car", "taxi", "train", "ticket"]):
        return "Travel"
    elif any(keyword in text for keyword in ["hotel", "stay", "room", "resort", "accommodation"]):
        return "Stay"
    else:
        return "Others"

# Store DataFrame to Azure SQL Database
def store_to_azure_sql(dataframe):
    try:
        conn = pymssql.connect(
            server="piosqlserverbd.database.windows.net",
            user="pio-admin",
            password="Poctest123#",
            database="PIOSqlDB"
        )
        cursor = conn.cursor()

        create_table_query = """
        IF NOT EXISTS (SELECT * FROM sysobjects WHERE name='Invoices' AND xtype='U')
        CREATE TABLE Invoices (
            EmployeeID NVARCHAR(50) NOT NULL PRIMARY KEY,
            InvoiceNumber NVARCHAR(255),
            Date NVARCHAR(255),
            Place NVARCHAR(255),
            Amount NVARCHAR(255),
            Category NVARCHAR(255),
            ApprovalStatus NVARCHAR(50) DEFAULT 'Pending'
        )
        """
        cursor.execute(create_table_query)

        cursor.execute("SELECT TOP 1 EmployeeID FROM Invoices ORDER BY EmployeeID DESC")
        last_id = cursor.fetchone()
        next_id = 0 if last_id is None else int(last_id[0]) + 1

        for _, row in dataframe.iterrows():
            category = identify_category(row["Invoice Details"])
            insert_query = """
            INSERT INTO Invoices (EmployeeID, InvoiceNumber, Date, Place, Amount, Category, ApprovalStatus)
            VALUES (%s, %s, %s, %s, %s, %s, %s)
            """
            cursor.execute(
                insert_query,
                (
                    f"{next_id:03d}",
                    row.get("Invoice Number", "")[:255],
                    row.get("Date", ""),
                    row.get("Place", ""),
                    row.get("Amount", ""),
                    category,
                    "Pending"
                )
            )
            next_id += 1

        conn.commit()
        conn.close()
        return "Data successfully stored in Azure SQL Database."
    except Exception as e:
        return f"Error storing data to database: {e}"

# Process image and extract details
def process_image(image_path):
    messages = [{
        "role": "user",
        "content": [
            {"type": "image", "image": image_path},
            {"type": "text", "text": (
                "Extract the following details from the invoice:\n"
                "- 'invoice_number'\n"
                "- 'date'\n"
                "- 'place'\n"
                "- 'amount' (monetary value in the relevant currency)\n"
                "- 'category' (based on the invoice type)"
            )}
        ]
    }]
    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    image_inputs, video_inputs = process_vision_info(messages)
    inputs = processor(text=[text], images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt")
    inputs = inputs.to(model.device)
    generated_ids = model.generate(**inputs, max_new_tokens=128)
    generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
    output_text = processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)
    return parse_details(output_text[0])

def parse_details(details):
    parsed_data = {
        "Invoice Number": None,
        "Date": None,
        "Place": None,
        "Amount": None,
        "Invoice Details": details
    }
    lines = details.split("\n")
    for line in lines:
        lower_line = line.lower()
        if "invoice" in lower_line:
            parsed_data["Invoice Number"] = line.split(":")[-1].strip()
        elif "date" in lower_line:
            parsed_data["Date"] = line.split(":")[-1].strip()
        elif "place" in lower_line:
            parsed_data["Place"] = line.split(":")[-1].strip()
        elif any(keyword in lower_line for keyword in ["total", "amount", "cost"]):
            parsed_data["Amount"] = line.split(":")[-1].strip()
    return parsed_data

@app.route('/extract', methods=['POST'])
def extract_invoice():
    image_path = request.json.get('image_path')
    extracted_data = process_image(image_path)
    df = pd.DataFrame([extracted_data])
    status = store_to_azure_sql(df)
    return jsonify({"data": extracted_data, "status": status})

if __name__ == '__main__':
    app.run(port=22)