Upload folder using huggingface_hub

config.py

@@ -431,7 +431,17 @@ class SanatanConfig:
             "title": "Chathusloki by Sri Alavandar",
             "output_dir": "./output/chathusloki",
             "collection_name": "chathusloki",
-            "unit": "slokam",
+            "unit": "verse",
+            "unit_field" : "sloka_number",
+            "field_mapping": {
+                "text": "sloka_devanagari",
+                "title": lambda doc: f"Verse {doc.get('verse','')}",
+                "unit_index": "sloka_number",
+                "transliteration": "sloka_english_transliteration",
+                "word_by_word_native": "meaning",
+                "author": lambda doc: f"Sri Aalavandhaar",
+                "relative_path": lambda doc: f"Chathusloki-{doc.get("verse","")}",
+            },
             "metadata_fields": [
                 {
                     "name": "sloka_number",
@@ -641,8 +651,12 @@ class SanatanConfig:
         canonical_doc["language"] = config.get("language")
         canonical_doc["unit"] = config.get("unit")
         canonical_doc["document"] = document_text
-        if canonical_doc["text"] == "-" or canonical_doc["text"] is None:
+        if (
+            canonical_doc.get("text", "-") == "-"
+            or canonical_doc.get("text", None) is None
+        ):
             canonical_doc["text"] = canonical_doc["document"]
+            canonical_doc["document"] = "-"
         canonical_doc["verse"] = resolve_field("verse")
 
         return canonical_doc

db.py

@@ -129,10 +129,17 @@ class SanatanDatabase:
             }
             Or a dict with "error" key if something went wrong.
         """
-        logger.info("Fetch document #%d from [%s]", index, collection_name)
+        logger.info("fetching document #%d from [%s]", index, collection_name)
         collection = self.chroma_client.get_or_create_collection(name=collection_name)
 
         try:
+            # show a sample data record
+            response = collection.get(
+                limit=1,
+                # offset=index,  # pagination via offset
+                include=["metadatas", "documents"],
+            )
+            print(response)
             response = collection.get(
                 limit=1,
                 # offset=index,  # pagination via offset

server.py

@@ -214,7 +214,7 @@ async def get_scripture(req: ScriptureRequest):
     raw_doc = SanatanDatabase().fetch_document_by_index(
         collection_name=config["collection_name"],
         index=req.unit_index,
-        unit_name=config["unit"]
+        unit_name=config.get("unit_field", config.get("unit"))
     )
 
     if not raw_doc or isinstance(raw_doc, str):