Upload folder using huggingface_hub

data/azhwars.json

@@ -42,7 +42,7 @@
     [
       "PAT",
       "Periyaazhwar",
-      "periyazvar thirumozhi"
+      "Periyazvar Thirumozhi"
     ],
     [
       "TP",
@@ -57,7 +57,7 @@
     [
       "MLT",
       "Poigaiazhwar",
-      "muthal thiruvanthathi"
+      "Muthal Thiruvanthathi"
     ],
     [
       "IT",

db.py

@@ -242,7 +242,7 @@ class SanatanDatabase:
                     include=["metadatas", "documents", "distances"],
                 )
         except Exception as e:
-            logger.error("Error in search: %s", e)
+            logger.error("Error in search: %s", e, exc_info=True)
             return chromadb.QueryResult(
                 documents=[],
                 ids=[],
@@ -642,11 +642,11 @@ class SanatanDatabase:
                 ]
                 batch_embeds = [embeddings[idx] for idx in batch_df.index]
 
-                collection.upsert(
+                collection.update(
                     ids=batch_ids,
-                    documents=batch_docs,
+                    # documents=batch_docs,
                     metadatas=batch_metas,
-                    embeddings=batch_embeds,
+                    # embeddings=batch_embeds,
                 )
 
             logger.info(

nalayiram_helper.py

@@ -10,12 +10,17 @@ class Pasuram:
 
 def get_standardized_prabandham_names() -> list[Pasuram]:
     """
-    Get a list of prabandham names along with the azhwars who authored them in divya_prabandham
+    Get a list of prabandham names along with the azhwars who authored them in divya_prabandham,
+    sorted by the prabandham name (3rd field, index 2).
     """
     with open("./data/azhwars.json", "r", encoding="utf-8") as f:
         azhwars = json.load(f) 
         header = azhwars[0]
         rows = azhwars[1:]
+
+        # Sort by 3rd field (index 2)
+        rows.sort(key=lambda row: row[2])
+
         final_azhwars = [Pasuram(**dict(zip(header, row))) for row in rows]
 
     return final_azhwars

tests/test_list_global_index.py

@@ -0,0 +1,17 @@
+import json
+import logging
+
+from db import SanatanDatabase
+from metadata import MetadataFilter, MetadataWhereClause
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    collection_name = "divya_prabandham"
+    database = SanatanDatabase()
+    result = database.chroma_client.get_collection(collection_name).get(include=["metadatas","documents"],where={"verse" : 21})
+
+    for index, metadata in enumerate(result["metadatas"]):
+        global_index = metadata.get("_global_index",-1)
+        verse = metadata["verse"]
+        print(f"global_index: {global_index} | verse: {verse} | metadata = {metadata}")