I have a code to map bounding boxes from json to image using ocr textract output which in csv format. But I am strugging to get correct bounding boxes for multi token entities. Multi token entity should be in one bounding box. I want some good clustering approch to address this issue. And a better solution to match the entity token from dataframe.

Please modify my code according to my requirements. Code should avoid taking values from anywhere in dataframe. It should always take the nearest word of other token in case of multi token entity. Code should be robust.

You have to first check the value of entity and then find its nearest word before printing the bounding box
import json
import pandas as pd
import cv2
from thefuzz import fuzz
from thefuzz import process
import numpy as np

def read_textract_output(csv_path):
    return pd.read_csv(csv_path)

def read_json_entities(json_path):
    with open(json_path, 'r') as file:
        return json.load(file)
    
def find_entity_bounding_boxes(entity_text, textract_df, image_size):
    entity_tokens = entity_text.split()
    results = pd.DataFrame()

    for token in entity_tokens:
        choices = textract_df["text"].dropna().tolist()
        best_match, score = process.extractOne(token, choices, scorer=fuzz.token_sort_ratio)
        if score > 70:
            best_matches_df = textract_df[textract_df["text"] == best_match]
            results = pd.concat([results, best_matches_df])

    bounding_boxes = []
    try:
        # Sort by line and word number to cluster words that are on the same line
        sorted_results = results.sort_values(by=["line_num", "word_num"])

        # Group the resulting bounding boxes by line number
        grouped_results = sorted_results.groupby("line_num")

        for _, group in grouped_results:
            # Calculate scaling factors based on image size and original image dimensions
            image_width, image_height = group.iloc[0]["image_width"], group.iloc[0]["image_height"]
            scale_x = image_size[0] / image_width
            scale_y = image_size[1] / image_height

            # Calculate the bounding box for the whole line
            min_left = np.min(group["left"])
            min_top = np.min(group["top"])
            max_right = np.max(group["left"] + group["width"])
            max_bottom = np.max(group["top"] + group["height"])
            bbox = (min_left * scale_x, min_top * scale_y, (max_right - min_left) * scale_x, (max_bottom - min_top) * scale_y)
            bounding_boxes.append(bbox)
            
    except Exception as e:
        print(f"An error occurred: {e}")

    return bounding_boxes
        
# def draw_bounding_boxes(image_path, entities, textract_df):
#     image = cv2.imread(image_path)
#     image_size = image.shape[1], image.shape[0]

#     for category, details in entities.items():
#         if category == "invoice_details" or category == "Payment Details" or category == "amounts_and_tax":
#             for entity, value in details.items():
#                 if value:
#                     bounding_boxes = find_entity_bounding_boxes(value, textract_df, image_size)
#                     try:
#                         for bbox in bounding_boxes:
#                             x, y, w, h = map(int, bbox)
#                             cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
#                     except:
#                         pass
#         else:
#             pass
def draw_bounding_boxes(image_path, entities, textract_df):
    image = cv2.imread(image_path)
    image_size = image.shape[1], image.shape[0]

    font = cv2.FONT_HERSHEY_SIMPLEX
    font_scale = 0.5
    font_color = (0, 0, 0)
    line_type = 2
    padding = 5  # Padding between the start of bbox and text

    for category, details in entities.items():
        if category == "invoice_details" or category == "Payment Details" or category == "amounts_and_tax":
            for entity, value in details.items():
                if value:
                    bounding_boxes = find_entity_bounding_boxes(value, textract_df, image_size)
                    try:
                        for bbox in bounding_boxes:
                            x, y, w, h = map(int, bbox)
                            cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
                            # Text placement calculation
                            text_position = (x, y - padding) if y - padding > 10 else (x, y + h + 20)
                            # Drawing the text
                            cv2.putText(image, entity, text_position, font, font_scale, font_color, line_type)
                    except:
                        pass
        else:
            pass
    cv2.imwrite('/home/ritik1s/Desktop/bbox_issues/temp_GPT/annotated_invoice.jpg', image)
    
    
IMAGE_PATH = '/home/ritik1s/Desktop/bbox_issues/temp_GPT/check.jpeg'
CSV_PATH = "/home/ritik1s/Desktop/bbox_issues/temp_GPT/check.csv"
JSON_PATH = "/home/ritik1s/Desktop/bbox_issues/temp_GPT/row_skip.json"

# Read Textract output and JSON entities
textract_df = read_textract_output(CSV_PATH)
entities = read_json_entities(JSON_PATH)

# Draw the bounding boxes
draw_bounding_boxes(IMAGE_PATH, entities, textract_df)