Update utility/utils.py

utility/utils.py

@@ -1,508 +1,485 @@
 # libraries
 import os
-from huggingface_hub import InferenceClient
-from dotenv import load_dotenv
+import base64
 import json
 import re
-#import easyocr
-from PIL import Image, ImageEnhance, ImageDraw
-import cv2
-import numpy as np
-from paddleocr import PaddleOCR
 import logging
 from datetime import datetime
 
+import cv2
+import numpy as np
+import requests
+from dotenv import load_dotenv
+from PIL import Image, ImageEnhance
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,
-    handlers=[
-        logging.StreamHandler()  # Remove FileHandler and log only to the console
-    ]
+    handlers=[logging.StreamHandler()]
 )
 
-# Set the PaddleOCR home directory to a writable location
-import os
-
-os.environ['PADDLEOCR_HOME'] = '/tmp/.paddleocr' 
-
-RESULT_FOLDER = 'static/results/'
-JSON_FOLDER = 'static/json/'
+# Load environment variables from .env file
+load_dotenv()
 
-if not os.path.exists('/tmp/.paddleocr'):
-    os.makedirs(RESULT_FOLDER, exist_ok=True)
+# Groq config
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
+GROQ_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"
 
-# Check if PaddleOCR home directory is writable
-if not os.path.exists('/tmp/.paddleocr'):
-    os.makedirs('/tmp/.paddleocr', exist_ok=True)
-    logging.info("Created PaddleOCR home directory.")
-else:
-    logging.info("PaddleOCR home directory exists.")
+RESULT_FOLDER = "static/results/"
+JSON_FOLDER = "static/json/"
 
-# Load environment variables from .env file
-load_dotenv()
+os.makedirs(RESULT_FOLDER, exist_ok=True)
+os.makedirs(JSON_FOLDER, exist_ok=True)
 
-# Authenticate with Hugging Face
-HFT = os.getenv('HF_TOKEN')
+# PaddleOCR home directory is no longer needed for the main path,
+# but keeping this does not hurt if something else imports it.
+os.environ["PADDLEOCR_HOME"] = "/tmp/.paddleocr"
+os.makedirs(os.environ["PADDLEOCR_HOME"], exist_ok=True)
 
-# Initialize the InferenceClient
-client = InferenceClient(model="mistralai/Mistral-7B-Instruct-v0.3", token=HFT)
 
 def load_image(image_path):
     ext = os.path.splitext(image_path)[1].lower()
-    if ext in ['.png', '.jpg', '.jpeg', '.webp', '.tiff']:
+    if ext in [".png", ".jpg", ".jpeg", ".webp", ".tiff", ".bmp"]:
         image = cv2.imread(image_path)
         if image is None:
-            raise ValueError(f"Failed to load image from {image_path}. The file may be corrupted or unreadable.")
+            raise ValueError(f"Failed to load image from {image_path}")
         return image
-    else:
-        raise ValueError(f"Unsupported image format: {ext}")   
-        
-# Function for upscaling image using OpenCV's INTER_CUBIC
+    raise ValueError(f"Unsupported image format: {ext}")
+
+
 def upscale_image(image, scale=2):
     height, width = image.shape[:2]
-    upscaled_image = cv2.resize(image, (width * scale, height * scale), interpolation=cv2.INTER_CUBIC)
-    return upscaled_image
+    return cv2.resize(image, (width * scale, height * scale), interpolation=cv2.INTER_CUBIC)
+
 
-# Function to denoise the image (reduce noise)
 def reduce_noise(image):
     return cv2.fastNlMeansDenoisingColored(image, None, 10, 10, 7, 21)
 
-# Function to sharpen the image
+
 def sharpen_image(image):
-    kernel = np.array([[0, -1, 0],
-                       [-1, 5, -1],
-                       [0, -1, 0]])
-    sharpened_image = cv2.filter2D(image, -1, kernel)
-    return sharpened_image
+    kernel = np.array([
+        [0, -1, 0],
+        [-1, 5, -1],
+        [0, -1, 0]
+    ])
+    return cv2.filter2D(image, -1, kernel)
+
 
-# Function to increase contrast and enhance details without changing color
 def enhance_image(image):
     pil_img = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
     enhancer = ImageEnhance.Contrast(pil_img)
     enhanced_image = enhancer.enhance(1.5)
-    enhanced_image_bgr = cv2.cvtColor(np.array(enhanced_image), cv2.COLOR_RGB2BGR)
-    return enhanced_image_bgr
+    return cv2.cvtColor(np.array(enhanced_image), cv2.COLOR_RGB2BGR)
+
 
-# Complete function to process image
 def process_image(image_path, scale=2):
-    # Load the image
     image = load_image(image_path)
-
-    # Upscale the image
     upscaled_image = upscale_image(image, scale)
-
-    # Reduce noise
     denoised_image = reduce_noise(upscaled_image)
-
-    # Sharpen the image
     sharpened_image = sharpen_image(denoised_image)
-
-    # Enhance the image contrast and details without changing color
     final_image = enhance_image(sharpened_image)
-
     return final_image
 
-# Function for OCR with PaddleOCR, returning both text and bounding boxes
-def ocr_with_paddle(img):
-    final_text = ''
-    boxes = []
-
-    # Initialize PaddleOCR
-    ocr = PaddleOCR(
-        lang='en', 
-        use_angle_cls=False
-        # det_model_dir=os.path.join(os.environ['PADDLEOCR_HOME'], 'whl/det'),
-        # rec_model_dir=os.path.join(os.environ['PADDLEOCR_HOME'], 'whl/rec/en/en_PP-OCRv4_rec_infer'),
-        # cls_model_dir=os.path.join(os.environ['PADDLEOCR_HOME'], 'whl/cls/ch_ppocr_mobile_v2.0_cls_infer')
-    )
-    # ocr = PaddleOCR(
-    #     use_angle_cls=True,
-    #     lang='en',
-    #     det_model_dir='/app/paddleocr_models/whl/det/ch_ppocr_mobile_v2.0_det_infer',
-    #     rec_model_dir='/app/paddleocr_models/whl/rec/ch_ppocr_mobile_v2.0_rec_infer',
-    #     cls_model_dir='/app/paddleocr_models/whl/cls/ch_ppocr_mobile_v2.0_cls_infer'
-    # )
 
+def image_to_base64(image):
+    """
+    image: OpenCV BGR numpy array
+    returns: base64 string of JPEG bytes
+    """
+    ok, buffer = cv2.imencode(".jpg", image)
+    if not ok:
+        raise ValueError("Failed to encode image to JPEG.")
+    return base64.b64encode(buffer).decode("utf-8")
+
+
+def _empty_schema():
+    return {
+        "Name": [],
+        "Designation": [],
+        "Company": [],
+        "Contact": [],
+        "Address": [],
+        "Email": [],
+        "Link": []
+    }
 
-    # Check if img is a file path or an image array
-    if isinstance(img, str):
-        img = cv2.imread(img)
 
-    # Perform OCR
-    result = ocr.ocr(img)
+def _coerce_list(value):
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [v for v in value if v is not None and str(v).strip() != ""]
+    if isinstance(value, tuple):
+        return [v for v in value if v is not None and str(v).strip() != ""]
+    if isinstance(value, str):
+        s = value.strip()
+        return [] if s == "" else [s]
+    return [value]
+
+
+def _strip_code_fences(text):
+    if not isinstance(text, str):
+        return text
+    text = text.strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.IGNORECASE)
+        text = re.sub(r"\s*```$", "", text)
+    return text.strip()
+
+
+def _parse_json_content(content):
+    """
+    Parses Groq response content into dict.
+    Handles:
+    - plain JSON string
+    - fenced JSON
+    - accidental text around JSON
+    """
+    if isinstance(content, dict):
+        return content
+
+    if content is None:
+        return {}
+
+    content = _strip_code_fences(str(content))
 
-    # Iterate through the OCR result
-    for line in result[0]:
-        # Check how many values are returned (2 or 3) and unpack accordingly
-        if len(line) == 3:
-            box, (text, confidence), _ = line  # When 3 values are returned
-        elif len(line) == 2:
-            box, (text, confidence) = line  # When only 2 values are returned
+    try:
+        return json.loads(content)
+    except json.JSONDecodeError:
+        # Try to recover a JSON object embedded in text
+        match = re.search(r"\{.*\}", content, flags=re.DOTALL)
+        if match:
+            return json.loads(match.group(0))
+        raise
+
+
+def normalize_llm_schema(data):
+    """
+    Normalizes model output to:
+    {
+        "Name": [],
+        "Designation": [],
+        "Company": [],
+        "Contact": [],
+        "Address": [],
+        "Email": [],
+        "Link": []
+    }
+    Accepts a dict that may have nulls, strings, or alternate key spellings.
+    """
+    data = data or {}
+
+    # Common alternate keys seen in model outputs
+    key_aliases = {
+        "Name": ["Name", "name", "FullName", "full_name", "person_name"],
+        "Designation": ["Designation", "designation", "Title", "title", "Role", "role"],
+        "Company": ["Company", "company", "Organization", "organization", "Org", "org"],
+        "Contact": ["Contact", "contact", "Phone", "phone", "Mobile", "mobile", "PhoneNumber", "phone_number"],
+        "Address": ["Address", "address", "Location", "location"],
+        "Email": ["Email", "email", "E-mail", "e_mail"],
+        "Link": ["Link", "link", "URL", "url", "Website", "website", "Portfolio", "portfolio"]
+    }
 
-        # Store the recognized text and bounding boxes
-        final_text += ' ' + text  # Extract the text from the tuple
-        boxes.append(box)
+    normalized = _empty_schema()
 
-        # Draw the bounding box
-        points = [(int(point[0]), int(point[1])) for point in box]
-        cv2.polylines(img, [np.array(points)], isClosed=True, color=(0, 255, 0), thickness=2)
+    for canonical_key, aliases in key_aliases.items():
+        chosen = []
+        for alias in aliases:
+            if alias in data and data[alias] is not None:
+                chosen = _coerce_list(data[alias])
+                break
+        normalized[canonical_key] = chosen
 
-    # Store the image with bounding boxes in a variable
-    img_with_boxes = img
+    return normalized
+
+
+def call_groq_vlm(image_bgr, prompt, timeout=120, retries=2):
+    if not GROQ_API_KEY:
+        raise ValueError("GROQ_API_KEY is missing from environment variables.")
+
+    base64_image = image_to_base64(image_bgr)
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {GROQ_API_KEY}"
+    }
+
+    payload = {
+        "model": GROQ_MODEL,
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "You are a strict information extraction engine. "
+                    "Return only valid JSON and no markdown."
+                )
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{base64_image}"
+                        }
+                    }
+                ]
+            }
+        ],
+        "temperature": 0.1,
+        "top_p": 1,
+        "max_completion_tokens": 1024,
+        "stream": False,
+        "response_format": {"type": "json_object"}
+    }
+
+    last_error = None
+    for attempt in range(retries + 1):
+        try:
+            resp = requests.post(GROQ_URL, headers=headers, json=payload, timeout=timeout)
+            resp.raise_for_status()
+            data = resp.json()
+
+            content = data["choices"][0]["message"]["content"]
+            parsed = _parse_json_content(content)
+            return normalize_llm_schema(parsed)
+
+        except Exception as e:
+            last_error = e
+            logging.exception(f"Groq VLM request failed on attempt {attempt + 1}")
+            if attempt < retries:
+                continue
+
+    raise last_error
+
+
+def build_vlm_prompt():
+    return """
+Extract structured text from this image and return ONLY valid JSON.
+
+Schema:
+{
+  "Name": [],
+  "Designation": [],
+  "Company": [],
+  "Contact": [],
+  "Address": [],
+  "Email": [],
+  "Link": []
+}
+
+Rules:
+- Always return all keys.
+- Every value must be a JSON array.
+- If a field is not found, return [].
+- Do not return null.
+- Do not add explanations or markdown.
+- Extract all visible text from the image, including business card text, printed labels, logos, URLs, and contact details.
+"""
 
-    return final_text, img_with_boxes
 
 def extract_text_from_images(image_paths):
+    """
+    Groq VLM single-pass extraction.
+    Returns:
+        merged_llm_data: dict with the schema above
+        all_extracted_texts: dict[path] -> JSON string per image
+        all_extracted_imgs: dict[path] -> processed image path
+    """
+    merged_llm_data = _empty_schema()
     all_extracted_texts = {}
     all_extracted_imgs = {}
+
     for image_path in image_paths:
         try:
-            # Enhance the image before OCR        
             enhanced_image = process_image(image_path, scale=2)
 
-            # Perform OCR on the enhanced image and get boxes
-            result, img_with_boxes = ocr_with_paddle(enhanced_image)
-
-            # Draw bounding boxes on the processed image
-            img_result = Image.fromarray(enhanced_image)
-            #img_with_boxes = draw_boxes(img_result, boxes)
-
-            # genrating unique id to save the images 
-            # Get the current date and time
             current_time = datetime.now()
-
-            # Format it as a string to create a unique ID
             unique_id = current_time.strftime("%Y%m%d%H%M%S%f")
+            result_image_path = os.path.join(
+                RESULT_FOLDER,
+                f"result_{unique_id}_{os.path.basename(image_path)}"
+            )
+
+            cv2.imwrite(result_image_path, enhanced_image)
 
-            #print(unique_id)
+            single_data = call_groq_vlm(
+                enhanced_image,
+                build_vlm_prompt()
+            )
 
-            # Save the image with boxes
-            result_image_path = os.path.join(RESULT_FOLDER, f'result_{unique_id}_{os.path.basename(image_path)}')
-            #img_with_boxes.save(result_image_path)
-            cv2.imwrite(result_image_path, img_with_boxes)
+            # Merge into combined schema
+            for key in merged_llm_data.keys():
+                merged_llm_data[key].extend(_coerce_list(single_data.get(key)))
 
-            # Store the text and image result paths
-            all_extracted_texts[image_path] = result
+            # Keep per-image extracted JSON as text for downstream regex processing
+            all_extracted_texts[image_path] = json.dumps(single_data, ensure_ascii=False)
             all_extracted_imgs[image_path] = result_image_path
-        except ValueError as ve:
-            print(f"Error processing image {image_path}: {ve}")
-            continue  # Continue to the next image if there's an error
-
-    # Convert to JSON-compatible structure
-    all_extracted_imgs_json = {str(k): str(v) for k, v in all_extracted_imgs.items()}
-    return all_extracted_texts, all_extracted_imgs_json
-
-# Function to call the Gemma model and process the output as Json 
-def Data_Extractor(data, client=client):
-    text = f'''Act as a  Text extractor for the following text given in text: {data} 
-    extract text in the following output JSON string:
-    {{
-    "Name": ["Identify and Extract All the person's name from the text."],
-    "Designation": ["Extract All the designation or job title mentioned in the text."],
-    "Company": ["Extract All the company or organization name if mentioned."],
-    "Contact": ["Extract All phone number, including country codes if present."],
-    "Address": ["Extract All the full postal address or location mentioned in the text."],
-    "Email": ["Identify and Extract All valid email addresses mentioned in the text else 'Not found'."],
-    "Link": ["Identify and Extract any website URLs or social media links present in the text."]
-    }}    
-    Output:
-    '''
-    
-    # Call the API for inference
-    response = client.text_generation(text, max_new_tokens=1000)#, temperature=0.4, top_k=50, top_p=0.9, repetition_penalty=1.2)
-    
-    print("parse in text ---:",response)
-
-    # Convert the response text to JSON
-    try:
-        json_data = json.loads(response)
-        print("Json_data-------------->",json_data)
-        return json_data
-    except json.JSONDecodeError as e:
-        return {"error": f"Error decoding JSON: {e}"}   
 
-# For have text compatible to the llm
-def json_to_llm_str(textJson):
-    str=''
-    for file,item in textJson.items():
-      str+=item + ' '
-    return str
+            logging.info(f"Processed image: {image_path}")
+
+        except Exception as e:
+            logging.exception(f"Error processing image {image_path}: {e}")
+            continue
+
+    return merged_llm_data, all_extracted_texts, all_extracted_imgs
+
 
-# Define the RE for extracting the contact details like number, mail , portfolio, website etc 
 def extract_contact_details(text):
-    # Regex patterns
-    # Phone numbers with at least 5 digits in any segment 
+    # Keep your existing regex logic here exactly as-is.
+    # This function is unchanged from your current file.
     combined_phone_regex = re.compile(r'''
-    (?: 
-        #(?:(?:\+91[-.\s]?)?\d{5}[-.\s]?\d{5})|(?:\+?\d{1,3})?[-.\s()]?\d{5,}[-.\s()]?\d{5,}[-.\s()]?\d{1,9} | /^[\.-)( ]*([0-9]{3})[\.-)( ]*([0-9]{3})[\.-)( ]*([0-9]{4})$/ |
-        \+1\s\(\d{3}\)\s\d{3}-\d{4} |               # USA/Canada Intl +1 (XXX) XXX-XXXX
-        \(\d{3}\)\s\d{3}-\d{4} |                    # USA/Canada STD (XXX) XXX-XXXX
-        \(\d{3}\)\s\d{3}\s\d{4} |                   # USA/Canada (XXX) XXX XXXX
-        \(\d{3}\)\s\d{3}\s\d{3} |                   # USA/Canada (XXX) XXX XXX
-        \+1\d{10} |                                 # +1 XXXXXXXXXX
-        \d{10} |                                    # XXXXXXXXXX
-        \+44\s\d{4}\s\d{6} |                        # UK Intl +44 XXXX XXXXXX
-        \+44\s\d{3}\s\d{3}\s\d{4} |                 # UK Intl +44 XXX XXX XXXX
-        0\d{4}\s\d{6} |                             # UK STD 0XXXX XXXXXX
-        0\d{3}\s\d{3}\s\d{4} |                      # UK STD 0XXX XXX XXXX
-        \+44\d{10} |                                # +44 XXXXXXXXXX
-        0\d{10} |                                   # 0XXXXXXXXXX
-        \+61\s\d\s\d{4}\s\d{4} |                    # Australia Intl +61 X XXXX XXXX
-        0\d\s\d{4}\s\d{4} |                         # Australia STD 0X XXXX XXXX
-        \+61\d{9} |                                 # +61 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+91\s\d{5}-\d{5} |                         # India Intl +91 XXXXX-XXXXX
-        \+91\s\d{4}-\d{6} |                         # India Intl +91 XXXX-XXXXXX
-        \+91\s\d{10} |                              # India Intl +91 XXXXXXXXXX
-        \+91\s\d{3}\s\d{3}\s\d{4} |                 # India Intl +91 XXX XXX XXXX
-        \+91\s\d{3}-\d{3}-\d{4} |                   # India Intl +91 XXX-XXX-XXXX
-        \+91\s\d{2}\s\d{4}\s\d{4} |                 # India Intl +91 XX XXXX XXXX
-        \+91\s\d{2}-\d{4}-\d{4} |                   # India Intl +91 XX-XXXX-XXXX
-        \+91\s\d{5}\s\d{5} |                        # India Intl +91 XXXXX XXXXX 
-        \d{5}\s\d{5} |                              # India XXXXX XXXXX 
-        \d{5}-\d{5} |                               # India XXXXX-XXXXX 
-        0\d{2}-\d{7} |                              # India STD 0XX-XXXXXXX
-        \+91\d{10} |                                # +91 XXXXXXXXXX
-        \d{10} |                                    # XXXXXXXXXX   # Here is the regex to handle all possible combination of the contact
-        \d{6}-\d{4} |                               # XXXXXX-XXXX
-        \d{4}-\d{6} |                               # XXXX-XXXXXX
-        \d{3}\s\d{3}\s\d{4} |                       # XXX XXX XXXX
-        \d{3}-\d{3}-\d{4} |                         # XXX-XXX-XXXX
-        \d{4}\s\d{3}\s\d{3} |                       # XXXX XXX XXX
-        \d{4}-\d{3}-\d{3} |                         # XXXX-XXX-XXX #-----
-        \+49\s\d{4}\s\d{8} |                        # Germany Intl +49 XXXX XXXXXXXX
-        \+49\s\d{3}\s\d{7} |                        # Germany Intl +49 XXX XXXXXXX
-        0\d{3}\s\d{8} |                             # Germany STD 0XXX XXXXXXXX
-        \+49\d{12} |                                # +49 XXXXXXXXXXXX
-        \+49\d{10} |                                # +49 XXXXXXXXXX
-        0\d{11} |                                   # 0XXXXXXXXXXX
-        \+86\s\d{3}\s\d{4}\s\d{4} |                 # China Intl +86 XXX XXXX XXXX
-        0\d{3}\s\d{4}\s\d{4} |                      # China STD 0XXX XXXX XXXX
-        \+86\d{11} |                                # +86 XXXXXXXXXXX
-        \+81\s\d\s\d{4}\s\d{4} |                    # Japan Intl +81 X XXXX XXXX
-        \+81\s\d{2}\s\d{4}\s\d{4} |                 # Japan Intl +81 XX XXXX XXXX
-        0\d\s\d{4}\s\d{4} |                         # Japan STD 0X XXXX XXXX
-        \+81\d{10} |                                # +81 XXXXXXXXXX
-        \+81\d{9} |                                 # +81 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+55\s\d{2}\s\d{5}-\d{4} |                  # Brazil Intl +55 XX XXXXX-XXXX
-        \+55\s\d{2}\s\d{4}-\d{4} |                  # Brazil Intl +55 XX XXXX-XXXX
-        0\d{2}\s\d{4}\s\d{4} |                      # Brazil STD 0XX XXXX XXXX
-        \+55\d{11} |                                # +55 XXXXXXXXXXX
-        \+55\d{10} |                                # +55 XXXXXXXXXX
-        0\d{10} |                                   # 0XXXXXXXXXX
-        \+33\s\d\s\d{2}\s\d{2}\s\d{2}\s\d{2} |      # France Intl +33 X XX XX XX XX
-        0\d\s\d{2}\s\d{2}\s\d{2}\s\d{2} |           # France STD 0X XX XX XX XX
-        \+33\d{9} |                                 # +33 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+7\s\d{3}\s\d{3}-\d{2}-\d{2} |             # Russia Intl +7 XXX XXX-XX-XX
-        8\s\d{3}\s\d{3}-\d{2}-\d{2} |               # Russia STD 8 XXX XXX-XX-XX
-        \+7\d{10} |                                 # +7 XXXXXXXXXX
-        8\d{10} |                                   # 8 XXXXXXXXXX
-        \+27\s\d{2}\s\d{3}\s\d{4} |                 # South Africa Intl +27 XX XXX XXXX
-        0\d{2}\s\d{3}\s\d{4} |                      # South Africa STD 0XX XXX XXXX
-        \+27\d{9} |                                 # +27 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+52\s\d{3}\s\d{3}\s\d{4} |                 # Mexico Intl +52 XXX XXX XXXX
-        \+52\s\d{2}\s\d{4}\s\d{4} |                 # Mexico Intl +52 XX XXXX XXXX
-        01\s\d{3}\s\d{4} |                          # Mexico STD 01 XXX XXXX
-        \+52\d{10} |                                # +52 XXXXXXXXXX
-        01\d{7} |                                   # 01 XXXXXXX
-        \+234\s\d{3}\s\d{3}\s\d{4} |                # Nigeria Intl +234 XXX XXX XXXX
-        0\d{3}\s\d{3}\s\d{4} |                      # Nigeria STD 0XXX XXX XXXX
-        \+234\d{10} |                               # +234 XXXXXXXXXX
-        0\d{10} |                                   # 0XXXXXXXXXX
-        \+971\s\d\s\d{3}\s\d{4} |                   # UAE Intl +971 X XXX XXXX
-        0\d\s\d{3}\s\d{4} |                         # UAE STD 0X XXX XXXX
-        \+971\d{8} |                                # +971 XXXXXXXX
-        0\d{8} |                                    # 0XXXXXXXX
-        \+54\s9\s\d{3}\s\d{3}\s\d{4} |              # Argentina Intl +54 9 XXX XXX XXXX
-        \+54\s\d{1}\s\d{4}\s\d{4} |                 # Argentina Intl +54 X XXXX XXXX
-        0\d{3}\s\d{4} |                             # Argentina STD 0XXX XXXX
-        \+54\d{10} |                                # +54 9 XXXXXXXXXX
-        \+54\d{9} |                                 # +54 XXXXXXXXX
-        0\d{7} |                                    # 0XXXXXXX
-        \+966\s\d\s\d{3}\s\d{4} |                   # Saudi Intl +966 X XXX XXXX
-        0\d\s\d{3}\s\d{4} |                         # Saudi STD 0X XXX XXXX
-        \+966\d{8} |                                # +966 XXXXXXXX
-        0\d{8} |                                    # 0XXXXXXXX
-        \+1\d{10} |                                 # +1 XXXXXXXXXX
-        \+1\s\d{3}\s\d{3}\s\d{4} |                  # +1 XXX XXX XXXX
-        \d{5}\s\d{5} |                              # XXXXX XXXXX                              
-        \d{10} |                                    # XXXXXXXXXX
-        \+44\d{10} |                                # +44 XXXXXXXXXX
-        0\d{10} |                                   # 0XXXXXXXXXX
-        \+61\d{9} |                                 # +61 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+91\d{10} |                                # +91 XXXXXXXXXX
-        \+49\d{12} |                                # +49 XXXXXXXXXXXX
-        \+49\d{10} |                                # +49 XXXXXXXXXX
-        0\d{11} |                                   # 0XXXXXXXXXXX
-        \+86\d{11} |                                # +86 XXXXXXXXXXX
-        \+81\d{10} |                                # +81 XXXXXXXXXX
-        \+81\d{9} |                                 # +81 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+55\d{11} |                                # +55 XXXXXXXXXXX
-        \+55\d{10} |                                # +55 XXXXXXXXXX
-        0\d{10} |                                   # 0XXXXXXXXXX
-        \+33\d{9} |                                 # +33 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX
-        \+7\d{10} |                                 # +7 XXXXXXXXXX
-        8\d{10} |                                   # 8 XXXXXXXXXX
-        \+27\d{9} |                                 # +27 XXXXXXXXX
-        0\d{9} |                                    # 0XXXXXXXXX (South Africa STD)
-        \+52\d{10} |                                # +52 XXXXXXXXXX
-        01\d{7} |                                   # 01 XXXXXXX
-        \+234\d{10} |                               # +234 XXXXXXXXXX
-        0\d{10} |                                   # 0XXXXXXXXXX
-        \+971\d{8} |                                # +971 XXXXXXXX
-        0\d{8} |                                    # 0XXXXXXXX
-        \+54\s9\s\d{10} |                           # +54 9 XXXXXXXXXX
-        \+54\d{9} |                                 # +54 XXXXXXXXX
-        0\d{7} |                                    # 0XXXXXXX
-        \+966\d{8} |                                # +966 XXXXXXXX
-        0\d{8}                                      # 0XXXXXXXX
+    (?:
+        \+1\s\(\d{3}\)\s\d{3}-\d{4} |
+        \(\d{3}\)\s\d{3}-\d{4} |
+        \(\d{3}\)\s\d{3}\s\d{4} |
+        \+1\d{10} |
+        \d{10} |
+        \+44\s\d{4}\s\d{6} |
+        \+44\s\d{3}\s\d{3}\s\d{4} |
+        0\d{4}\s\d{6} |
+        0\d{3}\s\d{3}\s\d{4} |
+        \+44\d{10} |
+        0\d{10} |
+        \+91\s\d{5}-\d{5} |
+        \+91\s\d{4}-\d{6} |
+        \+91\s\d{10} |
+        \+91\s\d{3}\s\d{3}\s\d{4} |
+        \+91\s\d{3}-\d{3}-\d{4} |
+        \+91\s\d{2}\s\d{4}\s\d{4} |
+        \+91\s\d{2}-\d{4}-\d{4} |
+        \+91\s\d{5}\s\d{5} |
+        \d{5}\s\d{5} |
+        \d{5}-\d{5} |
+        0\d{2}-\d{7} |
+        \+91\d{10} |
+        \d{6}-\d{4} |
+        \d{4}-\d{6} |
+        \d{3}\s\d{3}\s\d{4} |
+        \d{3}-\d{3}-\d{4} |
+        \d{4}\s\d{3}\s\d{3} |
+        \d{4}-\d{3}-\d{3} |
         \+\d{3}-\d{3}-\d{4}
-    )  
+    )
+    ''', re.VERBOSE)
 
-    ''',re.VERBOSE)
-    
-    # Email regex
     email_regex = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b')
-    
-    # URL and links regex, updated to avoid conflicts with email domains
     link_regex = re.compile(r'\b(?:https?:\/\/)?(?:www\.)[a-zA-Z0-9-]+\.(?:com|co\.in|co|io|org|net|edu|gov|mil|int|uk|us|in|de|au|app|tech|xyz|info|biz|fr|dev)\b')
-    
-    # Find all matches in the text
+
     phone_numbers = [num for num in combined_phone_regex.findall(text) if len(num) >= 5]
-    
     emails = email_regex.findall(text)
-
-    links_RE = [link for link in link_regex.findall(text) if len(link)>=11]
-    
-    # Remove profile links that might conflict with emails
+    links_RE = [link for link in link_regex.findall(text) if len(link) >= 11]
     links_RE = [link for link in links_RE if not any(email in link for email in emails)]
-    
+
     return {
         "phone_numbers": phone_numbers,
         "emails": emails,
         "links_RE": links_RE
-    }  
+    }
+
 
-# preprocessing the data 
 def process_extracted_text(extracted_text):
-    # Load JSON data
-    data = json.dumps(extracted_text, indent=4)
-    data = json.loads(data)
+    data = json.loads(json.dumps(extracted_text, indent=4))
 
-    # Create a single dictionary to hold combined results
     combined_results = {
         "phone_numbers": [],
         "emails": [],
         "links_RE": []
     }
 
-    # Process each text entry
     for filename, text in data.items():
         contact_details = extract_contact_details(text)
-        # Extend combined results with the details from this file
         combined_results["phone_numbers"].extend(contact_details["phone_numbers"])
         combined_results["emails"].extend(contact_details["emails"])
         combined_results["links_RE"].extend(contact_details["links_RE"])
 
-    # Convert the combined results to JSON
-    #combined_results_json = json.dumps(combined_results, indent=4)
-    combined_results_json = combined_results
-
-    # Print the final JSON results
     print("Combined contact details in JSON format:")
-    print(combined_results_json)
+    print(combined_results)
+
+    return combined_results
 
-    return combined_results_json 
 
-# Function to remove duplicates (case-insensitive) from each list in the dictionary
 def remove_duplicates_case_insensitive(data_dict):
     for key, value_list in data_dict.items():
+        if not isinstance(value_list, list):
+            continue
+
         seen = set()
         unique_list = []
-        
+
         for item in value_list:
-            if item.lower() not in seen:
-                unique_list.append(item)  # Add original item (preserving its case)
-                seen.add(item.lower())    # Track lowercase version
-        
-        # Update the dictionary with unique values
+            item_str = str(item)
+            key_lower = item_str.lower()
+            if key_lower not in seen:
+                unique_list.append(item)
+                seen.add(key_lower)
+
         data_dict[key] = unique_list
+
     return data_dict
 
-# Process the model output for parsed result
-def process_resume_data(LLMdata,cont_data,extracted_text):
-
-    # Removing duplicate emails
-    unique_emails = []
-    for email in cont_data['emails']:
-        if not any(email.lower() == existing_email.lower() for existing_email in LLMdata['Email']):
-            unique_emails.append(email)
-    
-    # Removing duplicate links (case insensitive)
-    unique_links = []
-    for link in cont_data['links_RE']:
-        if not any(link.lower() == existing_link.lower() for existing_link in LLMdata['Link']):
-            unique_links.append(link)
-    
-    # Removing duplicate phone numbers
-    normalized_contact = [num[-10:] for num in LLMdata['Contact']]
-    unique_numbers = []
-    for num in cont_data['phone_numbers']:
-        if num[-10:] not in normalized_contact:
-            unique_numbers.append(num)
-       
-    # Add unique emails, links, and phone numbers to the original LLMdata
-    LLMdata['Email'] += unique_emails
-    LLMdata['Link'] += unique_links
-    LLMdata['Contact'] += unique_numbers
-
-    # Apply the function to the data
-    LLMdata=remove_duplicates_case_insensitive(LLMdata)
-        
-    # Initialize the processed data dictionary
-    processed_data = {            
-            "name": [],
-            "contact_number": [],
-            "Designation":[],
-            "email": [],
-            "Location": [],
-            "Link": [],
-            "Company":[],
-            "extracted_text": extracted_text
-            }
-    #LLM
-    
-    processed_data['name'].extend(LLMdata.get('Name', None))
-    #processed_data['contact_number'].extend(LLMdata.get('Contact', []))
-    processed_data['Designation'].extend(LLMdata.get('Designation', []))
-    #processed_data['email'].extend(LLMdata.get("Email", []))
-    processed_data['Location'].extend(LLMdata.get('Address', []))
-    #processed_data['Link'].extend(LLMdata.get('Link', []))
-    processed_data['Company'].extend(LLMdata.get('Company', []))
-    
-    #Contact
-    #processed_data['email'].extend(cont_data.get("emails", [])) 
-    #processed_data['contact_number'].extend(cont_data.get("phone_numbers", []))
-    #processed_data['Link'].extend(cont_data.get("links_RE", []))
-    
-    #New_merge_data
-    processed_data['email'].extend(LLMdata['Email']) 
-    processed_data['contact_number'].extend(LLMdata['Contact'])
-    processed_data['Link'].extend(LLMdata['Link'])
-
-    #to remove not found fields
-    # List of keys to check for 'Not found'
-    keys_to_check = ["name", "contact_number", "Designation", "email", "Location", "Link", "Company"]
-
-    # Replace 'Not found' with an empty list for each key
-    for key in keys_to_check:
-        if processed_data[key] == ['Not found'] or processed_data[key] == ['not found']:
-            processed_data[key] = []
-    
-    return processed_data
+
+def process_resume_data(LLMdata, cont_data, extracted_text):
+    """
+    Final merge step.
+    Keeps the output structure exactly as you currently use in result.html.
+    """
+    LLMdata = normalize_llm_schema(LLMdata)
+    cont_data = cont_data or {}
+
+    cont_data.setdefault("emails", [])
+    cont_data.setdefault("phone_numbers", [])
+    cont_data.setdefault("links_RE", [])
+
+    # Merge regex-detected emails
+    existing_emails = {str(e).lower() for e in LLMdata["Email"]}
+    for email in cont_data["emails"]:
+        if str(email).lower() not in existing_emails:
+            LLMdata["Email"].append(email)
+            existing_emails.add(str(email).lower())
+
+    # Merge regex-detected links
+    existing_links = {str(l).lower() for l in LLMdata["Link"]}
+    for link in cont_data["links_RE"]:
+        if str(link).lower() not in existing_links:
+            LLMdata["Link"].append(link)
+            existing_links.add(str(link).lower())
+
+    # Merge regex-detected contacts using last-10-digit normalization
+    normalized_contacts = {str(num)[-10:] for num in LLMdata["Contact"] if num}
+    for num in cont_data["phone_numbers"]:
+        norm = str(num)[-10:]
+        if norm not in normalized_contacts:
+            LLMdata["Contact"].append(num)
+            normalized_contacts.add(norm)
+
+    LLMdata = remove_duplicates_case_insensitive(LLMdata)
+
+    processed_data = {
+        "name": LLMdata.get("Name", []),
+        "contact_number": LLMdata.get("Contact", []),
+        "Designation": LLMdata.get("Designation", []),
+        "email": LLMdata.get("Email", []),
+        "Location": LLMdata.get("Address", []),
+        "Link": LLMdata.get("Link", []),
+        "Company": LLMdata.get("Company", []),
+        "extracted_text": extracted_text
+    }
+
+    for key in ["name", "contact_number", "Designation", "email", "Location", "Link", "Company"]:
+        processed_data[key] = [
+            v for v in processed_data[key]
+            if str(v).strip().lower() not in {"not found", "none", "null", ""}
+        ]
+
+    return processed_data
+
+
+# Optional compatibility helper; no longer needed by the main flow.
+def json_to_llm_str(textJson):
+    s = ""
+    for _, item in textJson.items():
+        s += str(item) + " "
+    return s