I have a code understand the code and utilize the is_nearby function in process_multi_token_entity function.

import cv2
import pandas as pd
import json
from thefuzz import fuzz
from itertools import product
used_bounding_boxes = {}


def preprocess_entity(entity):
    try:
        token = entity.replace(",", "").strip()
        return token
    except:
        pass

def calculate_proximity_score(box_a, box_b):
    vertical_overlap = max(0, min(box_a["bottom"], box_b["bottom"]) - max(box_a["top"], box_b["top"]))
    vertical_distance = 0 if vertical_overlap > 0 else min(abs(box_a["top"] - box_b["bottom"]), abs(box_a["bottom"] - box_b["top"]))
    horizontal_overlap = max(0, min(box_a["right"], box_b["right"]) - max(box_a["left"], box_b["left"]))
    horizontal_distance = 0 if horizontal_overlap > 0 else abs(box_a["right"] - box_b["left"])
    return horizontal_distance + 2 * vertical_distance

def is_nearby(box_a, box_b, max_line_difference=1, max_distance=30):
    return calculate_proximity_score(box_a, box_b) <= max_distance + 2 * max_line_difference

def merge_boxes(boxes):
    min_left = min(box["left"] for box in boxes)
    max_right = max(box["right"] for box in boxes)
    min_top = min(box["top"] for box in boxes)
    max_bottom = max(box["bottom"] for box in boxes)
    return {"left": min_left, "right": max_right, "top": min_top, "bottom": max_bottom}

def find_potential_matches(dataframe, token, threshold=75):
    potential_matches = []
    for _, row in dataframe.iterrows():
        ocr_text = preprocess_entity(row["text"])
        score = fuzz.ratio(token, ocr_text)
        if score > threshold:
            potential_matches.append({
                "box": {"left": row["left"], "right": row["right"], "top": row["top"], "bottom": row["bottom"]},
                "score": score
            })
    return potential_matches

def find_best_sequence_heuristic(matches_list):
    if not matches_list or len(matches_list[0]) == 0:
        return []
    best_sequence = [min(matches_list[0], key=lambda match: match["score"])]
    for next_matches in matches_list[1:]:
        current_box = best_sequence[-1]["box"]
        next_best_match = min(next_matches, key=lambda match: calculate_proximity_score(current_box, match["box"]))
        best_sequence.append(next_best_match)
    return best_sequence

# def process_single_token_entity(dataframe, entity, threshold=75):
#     best_match = None
#     best_score = threshold
#     entity = preprocess_entity(entity)
#     for _, row in dataframe.iterrows():
#         ocr_text = preprocess_entity(row["text"])
#         score = fuzz.ratio(entity, ocr_text)
#         if score > best_score:
#             best_score = score
#             best_match = {
#                 "left": row["left"], "right": row["right"],
#                 "top": row["top"], "bottom": row["bottom"]
#             }
#     return best_match

def process_single_token_entity(dataframe, entity, threshold=75):
    global used_bounding_boxes
    best_match = None
    best_score = threshold
    entity = preprocess_entity(entity)

    if entity not in used_bounding_boxes:
        used_bounding_boxes[entity] = []

    for _, row in dataframe.iterrows():
        ocr_text = preprocess_entity(row['text'])
        score = fuzz.ratio(entity, ocr_text)
        current_box = {'left': row['left'], 'right': row['right'], 'top': row['top'], 'bottom': row['bottom']}

        if score > best_score and current_box not in used_bounding_boxes[entity]:
            best_score = score
            best_match = current_box

    if best_match:
        used_bounding_boxes[entity].append(best_match)

    return best_match

# def process_multi_token_entity(dataframe, entity, threshold=85):
#     tokens = entity.split()
#     all_potential_matches = [find_potential_matches(dataframe, token, threshold) for token in tokens]
#     if not all([matches for matches in all_potential_matches]):
#         return None
#     best_sequence = find_best_sequence_heuristic(all_potential_matches)
#     if best_sequence:
#         boxes_to_merge = [match["box"] for match in best_sequence]
#         return merge_boxes(boxes_to_merge)
#     return None

# def process_multi_token_entity(dataframe, entity, threshold=85):
#     global used_bounding_boxes
#     tokens = entity.split()
#     all_potential_matches = []
    
#     if entity not in used_bounding_boxes:
#         used_bounding_boxes[entity] = []

#     for token in tokens:
#         potential_matches = find_potential_matches(dataframe, token, threshold)
#         filtered_potential_matches = [match for match in potential_matches if match['box'] not in used_bounding_boxes[entity]]
#         all_potential_matches.append(filtered_potential_matches)

#     if not all([matches for matches in all_potential_matches]):
#         return None
    
#     best_sequence = find_best_sequence_heuristic(all_potential_matches)
#     if best_sequence :
#         boxes_to_merge = [match['box'] for match in best_sequence]
#         merged_box = merge_boxes(boxes_to_merge)
#         used_bounding_boxes[entity].append(merged_box)
#         return merged_box
    
#     return None

def box_overlap(box1, box2):
    """Check if there’s any overlap in any coordinate between two boxes."""
    return box1["left"] == box2["left"] or box1["right"] == box2["right"] 

def all_boxes_unique(sequence_boxes, used_boxes):
    """Ensure no part of the boxes in sequence_boxes overlaps with any box in used_boxes."""
    for seq_box in sequence_boxes:
        for used_box in used_boxes:
            if box_overlap(seq_box, used_box):
                return False
    return True

def get_next_best_sequence(all_potential_matches, previous_matches, entity):
    """
    Try to find the next best sequence of matches that hasn’t used any part of the bounding boxes.
    """
    # Flatten the list of used boxes for easier comparison.
    used_boxes = [box for sequence in previous_matches.get(entity, []) for box in sequence]

    for sequence in product(*all_potential_matches):
        sequence_boxes = [match["box"] for match in sequence]
        
        if all_boxes_unique(sequence_boxes, used_boxes):
            return sequence  # Found a sequence where no box part has been used before

    return None  # No unique sequence found

def process_multi_token_entity(dataframe, entity, threshold=85):
    global used_bounding_boxes

    # if entity not in used_bounding_boxes:
    #     used_bounding_boxes[entity] = []
    
    tokens = entity.split()
    all_potential_matches = [find_potential_matches(dataframe, token, threshold) for token in tokens]
    
    # Ensuring all tokens have at least one match
    if not all(matches for matches in all_potential_matches):
        return None
    
    # This assumes used_bounding_boxes[entity] holds lists of used sequences of boxes (not merged boxes)
    previous_matches = used_bounding_boxes.get(entity, [])
    next_best_sequence = get_next_best_sequence(all_potential_matches, used_bounding_boxes, entity)
    
    if next_best_sequence:
        new_boxes_sequence = [match["box"] for match in next_best_sequence]
        merged_box = merge_boxes(new_boxes_sequence)
        
        # If we found a new sequence, add it to the used sequences for this entity
        if entity not in used_bounding_boxes:
            used_bounding_boxes[entity] = []
        used_bounding_boxes[entity].append(new_boxes_sequence)
        
        return merged_box
    
    return None

def draw_bounding_boxes(image_path, bounding_boxes, entity_names):
    image = cv2.imread(image_path)
    font = cv2.FONT_HERSHEY_SIMPLEX
    for box, name in zip(bounding_boxes, entity_names):
        if box:
            cv2.rectangle(image, (box["left"], box["top"]), (box["right"], box["bottom"]), (0, 255, 0), 2)
            cv2.putText(image, name, (box["left"], max(box["top"] - 10, 0)), font, 0.5, (0, 0, 255), 2)
    cv2.imwrite("annotated_image_using_dp1119.jpg", image)
    
def main(json_path, csv_path, image_path):
    with open(json_path, "r") as f:
        data = json.load(f)
    dataframe = pd.read_csv(csv_path)
    bounding_boxes = []
    entity_names = []

    # Existing processing for non-special sections
    special_sections = ["amounts_and_tax","Payment Details"]  # Define special handling cases here

    for section in ["invoice_details", "Payment Details", "amounts_and_tax"]:
        entities = data.get(section, {})
        
        # Check if the current section needs special handling
        if section not in special_sections:
            for entity_name, entity_value in entities.items():
                entity_value_no_comma = preprocess_entity(entity_value)
                if " " in entity_value_no_comma:
                    box = process_multi_token_entity(dataframe, entity_value_no_comma)
                else:
                    box = process_single_token_entity(dataframe, entity_value_no_comma)
                if box:
                    bounding_boxes.append(box)
                    entity_names.append(entity_name)
        else:
            # Special handling for "amounts_and_tax" section
            reversed_dataframe = dataframe.iloc[::-1].reset_index(drop=True) # Reverse the dataframe
            for entity_name, entity_value in entities.items():
                entity_value_no_comma = preprocess_entity(entity_value)
                if " " in entity_value_no_comma:
                    # Use the reversed_dataframe for multi-token entities
                    box = process_multi_token_entity(reversed_dataframe, entity_value_no_comma)
                else:
                    # Use the reversed_dataframe for single-token entities
                    box = process_single_token_entity(reversed_dataframe, entity_value_no_comma)
                if box:
                    bounding_boxes.append(box)
                    entity_names.append(entity_name)

    draw_bounding_boxes(image_path, bounding_boxes, entity_names)


main("/home/ritik1s/Desktop/bbox_issues/temp_GPT/row_skip.json", "/home/ritik1s/Desktop/bbox_issues/temp_GPT/check.csv", "/home/ritik1s/Desktop/bbox_issues/temp_GPT/check.jpeg")