correct logs formating for dataviewer

climateqa/constants.py

@@ -97,7 +97,7 @@ DOCUMENT_METADATA_DEFAULT_VALUES = {
     "content": "",
     "reranking_score": 0.0,
     "query_used_for_retrieval": "",
-    "sources_used": [],
+    "sources_used": [""],
     "question_used": "",
     "index_used": ""
 }

climateqa/logging.py

@@ -7,7 +7,7 @@ import csv
 import pandas as pd
 import io
 from typing import TypedDict, List
-from constants import DOCUMENT_METADATA_DEFAULT_VALUES
+from climateqa.constants import DOCUMENT_METADATA_DEFAULT_VALUES
 
 
 def serialize_docs(docs:list)->list:
@@ -25,18 +25,24 @@ def serialize_docs(docs:list)->list:
     """
     new_docs = []
     for doc in docs:
-        new_doc = {}
-        new_doc["page_content"] = doc.page_content
-        new_doc["metadata"] = {}
+        # Make sure we have a clean doc format
+        new_doc = {
+            "page_content": doc.get("page_content", ""),
+            "metadata": {}
+        }
         
-        # Define default empty values by field type
-
-        
-        # Copy each metadata field, using default if missing
+        # Ensure all metadata fields exist with defaults if missing
         for field, default_value in DOCUMENT_METADATA_DEFAULT_VALUES.items():
-            new_doc["metadata"][field] = doc.metadata.get(field, default_value)
-            
+            new_value =  (doc.get("metadata", {}).get(field, default_value))
+            try:
+                new_doc["metadata"][field] = type(default_value)(new_value)
+            except:
+                new_doc["metadata"][field] = default_value
+
         new_docs.append(new_doc)
+        
+    if new_docs == []:
+        new_docs = [{"page_content": "No documents found", "metadata": DOCUMENT_METADATA_DEFAULT_VALUES}]
     return new_docs
 
 ## AZURE LOGGING - DEPRECATED