Upload folder using huggingface_hub

.gitignore

@@ -15,4 +15,7 @@ outputs/
 chromadb-store_20251112.zip
 chromadb-store_20251118.zip
 chroma_exports/
-chroma_exports.zip
+chroma_exports.zip
+chromadb-store-CORRUPTED/
+chromadb-store_BLOATED.zip
+chromadb-store_OLD.zip

copy_chromadb.py

@@ -102,7 +102,102 @@ db_config = {
         "source_db_path": "../bhagavata_purana_ai/chromadb_store",
         "source_collection_name": "bhagavata_purana",
         "destination_collection_name": "bhagavata_purana",
-    },    
+    },
+    "agnipuranam": {
+        "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "agnipuranam",
+    "destination_collection_name": "agnipuranam"
+  },
+  "bhavishyapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "bhavishyapuranam",
+    "destination_collection_name": "bhavishyapuranam"
+  },
+  "brahmandpuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "brahmandpuranam",
+    "destination_collection_name": "brahmandpuranam"
+  },
+  "brahmapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "brahmapuranam",
+    "destination_collection_name": "brahmapuranam"
+  },
+  "brahmavaivarthapurana": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "brahmavaivarthapurana",
+    "destination_collection_name": "brahmavaivarthapurana"
+  },
+  "garudapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "garudapuranam",
+    "destination_collection_name": "garudapuranam"
+  },
+  "harivanshapuraanam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "harivanshapuraanam",
+    "destination_collection_name": "harivanshapuraanam"
+  },
+  "kurmapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "kurmapuranam",
+    "destination_collection_name": "kurmapuranam"
+  },
+  "lingapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "lingapuranam",
+    "destination_collection_name": "lingapuranam"
+  },
+  "markandeypuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "markandeypuranam",
+    "destination_collection_name": "markandeypuranam"
+  },
+  "matsyapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "matsyapuranam",
+    "destination_collection_name": "matsyapuranam"
+  },
+  "naradapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "naradapuranam",
+    "destination_collection_name": "naradapuranam"
+  },
+  "padmapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "padmapuranam",
+    "destination_collection_name": "padmapuranam"
+  },
+  "shivapuraanam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "shivapuraanam",
+    "destination_collection_name": "shivapuraanam"
+  },
+  "skandapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "skandapuranam",
+    "destination_collection_name": "skandapuranam"
+  },
+  "vaamanapuraanam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "vaamanapuraanam",
+    "destination_collection_name": "vaamanapuraanam"
+  },
+  "vaayupuraanam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "vaayupuraanam",
+    "destination_collection_name": "vaayupuraanam"
+  },
+  "varahapuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "varahapuranam",
+    "destination_collection_name": "varahapuranam"
+  },
+  "vishnupuranam": {
+    "source_db_path": "../puranas_ai/chromadb_store",
+    "source_collection_name": "vishnupuranam",
+    "destination_collection_name": "vishnu_puranam_openai"
+  }
 }
 
 parser = argparse.ArgumentParser(description="My app with database parameter")

db.py

@@ -824,15 +824,26 @@ class SanatanDatabase:
         )
 
         # Fetch all records (keep embeddings for upsert)
-        try:
-            results = collection.get(include=["metadatas", "documents", "embeddings"])
-        except Exception as e:
-            logger.error(
-                "build_global_index_for_all_scriptures:%s Error getting data from chromadb",
-                scripture_name,
-                exc_info=True,
-            )
-            return
+        MAX_RETRIES = 3
+        RETRY_DELAY = 5  # seconds
+
+        for attempt in range(1, MAX_RETRIES + 1):
+            try:
+                results = collection.get(include=["metadatas", "documents", "embeddings"])
+                break  # success → exit loop
+            except Exception as e:
+                logger.error(
+                    "build_global_index_for_all_scriptures:%s Error getting data from chromadb (attempt %s/%s)",
+                    scripture_name,
+                    attempt,
+                    MAX_RETRIES,
+                    exc_info=True,
+                )
+
+                if attempt == MAX_RETRIES:
+                    # still failing after 3 attempts
+                    return
+                time.sleep(RETRY_DELAY)  # wait before retry
 
         ids = results["ids"]
         metadatas = results["metadatas"]

modules/config/__init__.py

@@ -1,26 +1,44 @@
+from modules.config.agnipuranam import agnipuranam_config
 from modules.config.bhagavat_gita import bhagavat_gita_config
 from modules.config.bhagavata_purana import bhagavata_purana_config
+from modules.config.bhavishyapuranam import bhavishyapuranam_config
 from modules.config.brahma_sutra import brahma_sutra_config
+from modules.config.brahmandpuranam import brahmandpuranam_config
+from modules.config.brahmapuranam import brahmapuranam_config
+from modules.config.brahmavaivarthapurana import brahmavaivarthapurana_config
 from modules.config.chathusloki import chathusloki_config
 from modules.config.desika_prabandham import desika_prabandham_config
 from modules.config.divya_prabandham import divya_prabandham_config
 from modules.config.divya_prabandham_taniyans import divya_prabandham_taniyans_config
+from modules.config.garudapuranam import garudapuranam_config
+from modules.config.harivanshapuraanam import harivanshapuraanam_config
 from modules.config.kamba_ramayanam import kamba_ramayanam_config
 from modules.config.kamba_ramayanam_en import kamba_ramayanam_en_config
 from modules.config.katakam import katakam_config
+from modules.config.kurmapuranam import kurmapuranam_config
+from modules.config.lingapuranam import lingapuranam_config
+from modules.config.markandeypuranam import markandeypuranam_config
+from modules.config.matsyapuranam import matsyapuranam_config
+from modules.config.naradapuranam import naradapuranam_config
 from modules.config.narayaneeyam import narayaneeyam_config
+from modules.config.padmapuranam import padmapuranam_config
 from modules.config.pancha_sooktham import pancha_sooktham_config
 from modules.config.raghuveera_gadhyam import raghuveera_gadhyam_config
 from modules.config.shanthi_panchakam import shanthi_panchakam_config
+from modules.config.shivapuraanam import shivapuraanam_config
 from modules.config.shukla_yajur_vedam import shukla_yajur_vedam_config
+from modules.config.skandapuranam import skandapuranam_config
 from modules.config.sri_stavam import sri_stavam_config
 from modules.config.sri_vachana_bhushanam import sri_vachana_bhushanam_config
 from modules.config.taitriya_aranyakam import taitriya_aranyakam_config
 from modules.config.taitriya_brahmanam import taitriya_brahmanam_config
 from modules.config.taitriya_samhitha import taitriya_samhitha_config
 from modules.config.taitriya_upanishad import taitriya_upanishad_config
+from modules.config.vaamanapuraanam import vaamanapuraanam_config
+from modules.config.vaayupuraanam import vaayupuraanam_config
 from modules.config.valmiki_ramayanam import valmiki_ramayanam_config
-from modules.config.vishnu_puranam import vishnu_puranam_config
+from modules.config.varahapuranam import varahapuranam_config
+from modules.config.vishnu_puranam import vishnupuranam_config
 from modules.config.vishnu_sahasranamam import vishnu_sahasranamam_config
 from modules.config.yt_metadata import yt_metadata_config
 
@@ -43,7 +61,7 @@ scripture_configurations = [
     taitriya_aranyakam_config,
     taitriya_upanishad_config,
     valmiki_ramayanam_config,
-    vishnu_puranam_config,
+    vishnupuranam_config,
     vishnu_sahasranamam_config,
     yt_metadata_config,
     brahma_sutra_config,
@@ -51,4 +69,22 @@ scripture_configurations = [
     desika_prabandham_config,
     raghuveera_gadhyam_config,
     narayaneeyam_config,
+    agnipuranam_config,
+    bhavishyapuranam_config,
+brahmandpuranam_config,
+brahmapuranam_config,
+brahmavaivarthapurana_config,
+garudapuranam_config,
+harivanshapuraanam_config,
+kurmapuranam_config,
+lingapuranam_config,
+markandeypuranam_config,
+matsyapuranam_config,
+naradapuranam_config,
+padmapuranam_config,
+shivapuraanam_config,
+skandapuranam_config,
+vaamanapuraanam_config,
+vaayupuraanam_config,
+varahapuranam_config,    
 ]

modules/config/agnipuranam.py

@@ -0,0 +1,72 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+agnipuranam_config = {
+    "name": "agnipuranam",
+    "title": "Agni Puranam",
+    "banner_url": convert_drive_url_to_direct(
+        "https://drive.google.com/file/d/1YjOR3wLNBbNovKC01aUXHa1osbjgLYbw/view?usp=drive_link"
+    ),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/agnipuranam",
+    "collection_name": "agnipuranam",
+    "collection_embedding_fn": "openai",
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/agnipuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [
+            {
+                "name": "Sanskritam",
+                "url": "https://www.sanskritam.world/puranas",
+                "role": "Data provider",
+            },
+        ], "audio": [], "video": []},
+}

modules/config/bhavishyapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+bhavishyapuranam_config = {
+    "name": "bhavishyapuranam",
+    "title": "Bhavishya Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1ENd6Es_Ng7mj5ofoqmT2UGCbJM8ZFFVl/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/bhavishyapuranam",
+    "collection_name": "bhavishyapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/bhavishyapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/brahmandpuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+brahmandpuranam_config = {
+    "name": "brahmandpuranam",
+    "title": "Brahmanda Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1r0fV1AqASV0LsipTOPzJ7hmgswT5hNBZ/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/brahmandpuranam",
+    "collection_name": "brahmandpuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/brahmandpuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/brahmapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+brahmapuranam_config = {
+    "name": "brahmapuranam",
+    "title": "Brahma Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1CQn25LkeekY9c592tnzo8Msv0Ib8XiiE/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/brahmapuranam",
+    "collection_name": "brahmapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/brahmapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/brahmavaivarthapurana.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+brahmavaivarthapurana_config = {
+    "name": "brahmavaivarthapurana",
+    "title": "Brahma Vaivarta Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1SRLUvGvJaQ3SUaoEQMgy0ACTPGh1Sw6S/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/brahmavaivarthapurana",
+    "collection_name": "brahmavaivarthapurana",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/brahmavaivarthapurana.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/garudapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+garudapuranam_config = {
+    "name": "garudapuranam",
+    "title": "Garuda Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1jhGecKwco5RzGQzSVa8PiuFKJ26LHJ0p/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/garudapuranam",
+    "collection_name": "garudapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/garudapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/harivanshapuraanam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+harivanshapuraanam_config = {
+    "name": "harivanshapuraanam",
+    "title": "Harivansha Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uWkzBJWin19aS7_TE545E5cwOXGpTPrx/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/harivanshapuraanam",
+    "collection_name": "harivanshapuraanam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/harivanshapuraanam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/kurmapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+kurmapuranam_config = {
+    "name": "kurmapuranam",
+    "title": "Kurma Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1Y1epvVgGzyKlWSFEI2ep8bNwT6G7G6II/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/kurmapuranam",
+    "collection_name": "kurmapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/kurmapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/lingapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+lingapuranam_config = {
+    "name": "lingapuranam",
+    "title": "Linga Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/12Jg8WOnwwp2dCvFLKf2UN1LrFc4bpK7Q/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/lingapuranam",
+    "collection_name": "lingapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/lingapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/markandeypuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+markandeypuranam_config = {
+    "name": "markandeypuranam",
+    "title": "Markandeya Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/101ibf90vyBpKdcMQCvooQ30JEBL2XMlm/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/markandeypuranam",
+    "collection_name": "markandeypuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/markandeypuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/matsyapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+matsyapuranam_config = {
+    "name": "matsyapuranam",
+    "title": "Matsya Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1DxlpvcRqHGTFsE0tDgxThzUy-F0gmlMb/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/matsyapuranam",
+    "collection_name": "matsyapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/matsyapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/naradapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+naradapuranam_config = {
+    "name": "naradapuranam",
+    "title": "Narada Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1M-qDqwo9iCweONGDzhqUyjYcjJFgWpRd/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/naradapuranam",
+    "collection_name": "naradapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/naradapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/padmapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+padmapuranam_config = {
+    "name": "padmapuranam",
+    "title": "Padma Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1BC5tIabDwAFw4-UmXdZcae3FrI7vR3ek/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/padmapuranam",
+    "collection_name": "padmapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/padmapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/shivapuraanam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+shivapuraanam_config = {
+    "name": "shivapuraanam",
+    "title": "Shiva Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1kgLfcskt00H_lkJtSgf8FDyJqiikl3Eu/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/shivapuraanam",
+    "collection_name": "shivapuraanam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/shivapuraanam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/skandapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+skandapuranam_config = {
+    "name": "skandapuranam",
+    "title": "Skanda Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1J-yM-vFJFye0yV2oOsUAo23yv7PBfy2G/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/skandapuranam",
+    "collection_name": "skandapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/skandapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/vaamanapuraanam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+vaamanapuraanam_config = {
+    "name": "vaamanapuraanam",
+    "title": "Vaamana Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1xnRARh9wnfQ7oy7GxSr7-1mKvRj8Iq64/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/vaamanapuraanam",
+    "collection_name": "vaamanapuraanam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/vaamanapuraanam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/vaayupuraanam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+vaayupuraanam_config = {
+    "name": "vaayupuraanam",
+    "title": "Vaayu Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1nQqZKmGdLnFxSDcqsaq7PP5na1SZQ23h/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/vaayupuraanam",
+    "collection_name": "vaayupuraanam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/vaayupuraanam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/varahapuranam.py

@@ -0,0 +1,65 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+varahapuranam_config = {
+    "name": "varahapuranam",
+    "title": "Varaaha Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1jAZe95sUSH5iuNw2wdQKW7cLcf37ArgK/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/varahapuranam",
+    "collection_name": "varahapuranam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
+    "metadata_fields": [
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
+    ],
+
+    "pdf_path": "./data/varahapuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
+}

modules/config/vishnu_puranam.py

@@ -1,46 +1,65 @@
 from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
 
-
-vishnu_puranam_config = {
-    "name": "vishnu_puranam",
-    "title": "Sri Vishnu Puranam",
-    "category" : ScriptureCategoryConstants.PURANAM,
-    "output_dir": "./output/vishnu_puranam",
+vishnupuranam_config = {
+    "name": "vishnupuranam",
+    "title": "Vishnu Puranam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1ny6M0c6jfCLtYRaROffb1erRc3u8WB2u/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.PURANAM,
+    "output_dir": "./output/vishnupuranam",
     "collection_name": "vishnu_puranam_openai",
     "collection_embedding_fn": "openai",
-    "unit": "page",
+
+    "unit": "sloka",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: (
+            " | ".join(
+                [
+                    doc[key]
+                    for key in sorted(
+                        [k for k in doc.keys() if k.startswith("title_") and doc[k]],
+                        key=lambda x: int(x.split("_")[1])
+                    )
+                ]
+            )
+            + f" || {doc.get('_global_index', '')} ||"
+        )
+    },
+
     "metadata_fields": [
         {
-            "name": "file",
-            "label": "File Name",
-            "datatype": "str",
-            "description": "name of the file from which the information was extracted",
-        },
-        {
-            "name": "page",
+            "name": "_global_index",
             "datatype": "int",
-            "label": "Page Number",
-            "description": "Page number from the source",
+            "label": "Verse Index",
+            "description": "Absolute verse index",
             "show_as_filter": True,
             "is_unique": True,
         },
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter",
+            "description": "Original chapter title",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit",
+            "description": "Verse text in Sanskrit",
+        },
     ],
-    "pdf_path": "./data/vishnu_puranam.pdf",
-    "source": "https://dn720005.ca.archive.org/0/items/vishnu-purana-sanskrit-english-ocr/VISHNU-PURANA-Sanskrit-English-OCR.pdf",
-    "language": "san+eng",
-    "example_labels": [
-        "Vishnu's form",
-        "About the five elements",
-        "About Garuda",
-        "Weapons of Vishnu",
-        "Vishnu's form (all scriptures)",
-    ],
-    "examples": [
-        "describe Vishnu's form as defined in vishnu puranam",
-        "five elements and their significance as per vishnu puranam",
-        "What is the significance of Garuda? Show some verses from vishnu puranam that describe him.",
-        "What weapons does Vishnu hold as mentioned in vishnu puranam?",
-        "How is the form of Vishnu described across the scriptures?",
-    ],
+
+    "pdf_path": "./data/vishnupuranam.pdf",
+    "source": "",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
     "llm_hints": [],
+    "credits": {"art": [], "data": [], "audio": [], "video": []},
 }