Upload folder using huggingface_hub

config.py

@@ -1,1231 +1,11 @@
 from metadata import MetadataWhereClause
 from typing import List, Dict
 
-from modules.scripture_helpers.bhagavat_gita_helper import (
-    get_bhagavat_gita_chapters,
-    get_chapter_order_from_bhagavat_gita,
-)
-from modules.scripture_helpers.kamba_ramayanam_helper import (
-    get_list_of_kandams,
-    get_list_of_padalams,
-    get_list_of_padalams_in_tamil,
-)
-from modules.scripture_helpers.pancha_sooktham_helper import (
-    get_chapter_order_from_pancha_sooktham,
-    get_pancha_sooktham_chapters,
-)
-from modules.scripture_helpers.shanthi_panchakam_helper import (
-    get_chapter_order_from_shanthi_panchakam,
-    get_shanthi_panchakam_chapters,
-)
-from modules.scripture_helpers.taitriya_upanishad_helper import (
-    get_chapter_order_from_taitriya_upanishad,
-    get_taitriya_upanishad_chapters,
-)
-from modules.scripture_helpers.vishnu_sahasranamam_helper import (
-    get_chapter_order_from_sahasranamam,
-    get_chapters_from_sahasranamam,
-)
-import nalayiram_helper
-
+from modules.config import scripture_configurations
 
 class SanatanConfig:
     dbStorePath: str = "./chromadb-store"
-    scriptures = [
-        {
-            "name": "vishnu_puranam",
-            "title": "Sri Vishnu Puranam",
-            "output_dir": "./output/vishnu_puranam",
-            "collection_name": "vishnu_puranam_openai",
-            "collection_embedding_fn": "openai",
-            "unit": "page",
-            "metadata_fields": [
-                {
-                    "name": "file",
-                    "label": "File Name",
-                    "datatype": "str",
-                    "description": "name of the file from which the information was extracted",
-                },
-                {
-                    "name": "page",
-                    "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/vishnu_puranam.pdf",
-            "source": "https://dn720005.ca.archive.org/0/items/vishnu-purana-sanskrit-english-ocr/VISHNU-PURANA-Sanskrit-English-OCR.pdf",
-            "language": "san+eng",
-            "example_labels": [
-                "Vishnu's form",
-                "About the five elements",
-                "About Garuda",
-                "Weapons of Vishnu",
-                "Vishnu's form (all scriptures)",
-            ],
-            "examples": [
-                "describe Vishnu's form as defined in vishnu puranam",
-                "five elements and their significance as per vishnu puranam",
-                "What is the significance of Garuda? Show some verses from vishnu puranam that describe him.",
-                "What weapons does Vishnu hold as mentioned in vishnu puranam?",
-                "How is the form of Vishnu described across the scriptures?",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "shukla_yajur_vedam",
-            "title": "Shukla Yajur Vedam",
-            "output_dir": "./output/shukla_yajur_vedam",
-            "collection_name": "shukla_yajur_vedam",
-            "unit": "page",
-            "metadata_fields": [
-                {
-                    "name": "file",
-                    "label": "File Name",
-                    "datatype": "str",
-                    "description": "name of the file from which the information was extracted",
-                },
-                {
-                    "name": "page",
-                    "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/shukla-yajur-veda.pdf",
-            "source": "https://www.thearyasamaj.org/uploads/book/2014/04/R1sSjG_eLb_sub_406_yajurveda.pdf",
-            "language": "san+eng",
-            "example_labels": [
-                "About Vedam",
-                "About the five elements",
-                "About Brahma",
-            ],
-            "examples": [
-                "Gist of Shukla Yajur Vedam. Give me some sanskrit verses.",
-                "What is the significance of fire and water. show some sanskrit verses",
-                "Brahma",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "bhagavat_gita",
-            "title": "Bhagavat Gita",
-            "output_dir": "./output/bhagavat_gita",
-            "collection_name": "bhagavat_gita_openai",
-            "collection_embedding_fn": "openai",
-            "unit": "verse",
-            "unit_field": "relative_verse_number",
-            "chapter_order": lambda: get_chapter_order_from_bhagavat_gita(),
-            "audio_storage": "external",
-            "field_mapping": {
-                "text": "sanskrit",
-                "title": "verse_title",
-                "unit_index": "relative_verse_number",
-                "transliteration": "transliteration",
-                "word_by_word_native": "word_by_word_meaning",
-                "translation": "translation",
-                "author": lambda doc: f"Maharishi Vedvyas Ji",
-                "chapter_name": "chapter_title",
-                "relative_path": lambda doc: f"{doc.get("chapter_title","")}-{doc.get("relative_verse_number","")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "chapter_title",
-                    "datatype": "str",
-                    "label": "Chapter Name",
-                    "description": "Name of the Chapter",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_bhagavat_gita_chapters(),
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "translation",
-                    "label": "English Translation",
-                    "datatype": "str",
-                    "description": "The english translation.",
-                },
-                {
-                    "name": "transliteration",
-                    "label": "English Transliteration",
-                    "datatype": "str",
-                    "description": "The english transliteration.",
-                },
-                {
-                    "name": "relative_verse_number",
-                    "label": "Relative Verse Number",
-                    "datatype": "int",
-                    "description": "The relative verse number of the sloka inside the chapter.",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/bhagavat_gita.pdf",
-            "source": "https://vivekavani.com/",
-            "language": "san+eng",
-            "example_labels": [
-                "About Arjuna",
-                "About Karma",
-                "About birth and death",
-                "About the battle field",
-                "About Krishna's form",
-                "Krishna's Teachings",
-            ],
-            "examples": [
-                "Show some verses where Krishna advises Arjuna",
-                "What does Krishna say about Karma",
-                "What does Krishna say about birth and death",
-                "describe the battle field",
-                "How did Arjuna respond upon witnessing Krishna’s Vishwarupa?"
-                "What teachings did Krishna share in the Gita?",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "valmiki_ramayanam",
-            "title": "Valmiki Ramayanam",
-            "output_dir": "./output/valmiki_ramayanam",
-            "collection_name": "valmiki_ramayanam_openai",
-            "collection_embedding_fn": "openai",
-            "unit": "page",
-            "metadata_fields": [
-                {
-                    "name": "file",
-                    "label": "File Name",
-                    "datatype": "str",
-                    "description": "name of the file from which the information was extracted",
-                },
-                {
-                    "name": "page",
-                    "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/valmiki_ramayanam.pdf",
-            "source": "https://ia800509.us.archive.org/28/items/valmiki-ramayana-gita-press-english/Valmiki%20Ramayana%20Gita%20Press%20English.pdf",
-            "language": "san+eng",
-            "example_labels": [
-                "About Jatayu",
-                "About Hanuman",
-                "About Vali",
-                "About Sita",
-                "About Ravana",
-                "A slokam by name",
-                "Vibheeshana sharanagathi slokam",
-            ],
-            "examples": [
-                "What is the significance of Jatayu? show some sanskrit verses to support the argument",
-                "Show some verses where Hanuman is mentioned",
-                "How did Rama kill Vali",
-                "How was Sita abducted",
-                "How did Rama kill Ravana?",
-                "explain sakrudeva prapannaaya shlokam in ramayana",
-                "give the shlokam in ramayanam that vibheeshana uses to perform sharanagathi to rama, give the sanskrit shlokam and its meaning",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "vishnu_sahasranamam",
-            "title": "Vishnu Sahasranamam",
-            "output_dir": "./output/vishnu_sahasranamam",
-            "collection_name": "vishnu_sahasranamam_openai",
-            "collection_embedding_fn": "openai",
-            "unit": "verse",
-            "chapter_order": lambda: get_chapter_order_from_sahasranamam(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "title": lambda doc: f"Verse {doc.get('verse','')}",
-                "unit_index": "verse",
-                "transliteration": "transliteration",
-                "translation": "translation",
-                "author": lambda doc: f"Sage Vyasa",
-                "chapter_name": "chapter",
-                "relative_path": lambda doc: f"{doc.get("chapter","")}-{doc.get("verse","")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "chapter",
-                    "datatype": "str",
-                    "label": "Chapter Name",
-                    "description": "Name of the Chapter",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_chapters_from_sahasranamam(),
-                    "is_unique": True,
-                },
-                {
-                    "name": "page_number",
-                    "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "translation",
-                    "label": "English Translation",
-                    "datatype": "str",
-                    "description": "The english translation.",
-                },
-                {
-                    "name": "transliteration",
-                    "label": "English Transliteration",
-                    "datatype": "str",
-                    "description": "The english transliteration.",
-                },
-                {
-                    "name": "verse",
-                    "label": "Verse Number",
-                    "datatype": "int",
-                    "description": "The verse number of the sloka.",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/vishnu_sahasranamam.pdf",
-            "source": "https://www.swami-krishnananda.org/vishnu/Sri_Vishnu_Sahasranama_Stotram.pdf",
-            "language": "san+eng",
-            "example_labels": ["Vanamali", "1000 names", "Sanskrit text search"],
-            "examples": [
-                "Vanamali",
-                "Show some of the 1000 names of Vishnu along with their meaning",
-                "show the verse that begins with शुक्लाम्बरधरं",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "divya_prabandham",
-            "title": "4000 Divya Prabandham",
-            "output_dir": "./output/divya_prabandham",
-            "collection_name": "divya_prabandham",
-            "collection_embedding_fn": "openai",
-            "unit": "pasuram",
-            "unit_field" : "verse",
-            "credits": "We owe deep gratitude to uveda.org for providing such beautifully curated data. 🙏 This section would not have been possible without the dedicated efforts of the Uveda team.",
-            "field_mapping": {
-                "text": "pasuram_ta",
-                "title": lambda doc: f"{doc.get('prabandham_name','')} {doc.get('chapter','')}-{doc.get('decade','')}:{doc.get('position_in_chapter','')}",
-                "location": "divya_desams",
-                "word_by_word_native": "wbw_ta",
-                "unit_index": "verse",
-                "transliteration": "pasuram_en",
-                "reference_link": "html_url",
-                "author": "azhwar_name",
-                "chapter_name": "prabandham_name",
-                "relative_path": lambda doc: "-".join(
-                    filter(
-                        None,
-                        [
-                            doc.get("prabandham_name", ""),
-                            *(
-                                str(doc.get(k))
-                                for k in ["decade", "chapter", "position_in_chapter"]
-                                if doc.get(k, -1) != -1
-                            ),
-                        ],
-                    )
-                ),
-            },
-            "metadata_fields": [
-                {
-                    "name": "prabandham_code",
-                    "label": "Prabandham Code",
-                    "datatype": "str",
-                    "description": "contains the short prabandham_code. e.g. `TPL` for `Thiruppallandu`",
-                },
-                {
-                    "name": "prabandham_name",
-                    "label": "Prabandham Name",
-                    "datatype": "str",
-                    "description": "contains the prabandham name. e.g. `Thiruppallandu`",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: [
-                        p.prabandham_name
-                        for p in nalayiram_helper.get_standardized_prabandham_names()
-                    ],
-                },
-                {
-                    "name": "azhwar_name",
-                    "label": "Azhwar Name",
-                    "datatype": "str",
-                    "description": "contains the azhwar name. e.g. `Thirumangai Azhwar`",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: [
-                        azhwar_name
-                        for azhwar_name in nalayiram_helper.get_standardized_azhwar_names()
-                    ],
-                },
-                {
-                    "name": "divya_desams",
-                    "label": "Divya Desams",
-                    "datatype": "str",
-                    "description": "comma separated list of divya desams. e.g. Thiruneermalai,Thiruvallikkeni.",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: [
-                        divya_desam_name
-                        for divya_desam_name in nalayiram_helper.get_standardized_divya_desam_names()
-                    ],
-                },
-                {
-                    "name": "title",
-                    "label": "Pasuram Title",
-                    "datatype": "str",
-                    "description": (
-                        "Exact title of a pasuram in one of the following formats:\n"
-                        "1. '{prabandham_code} {decade}.{chapter}.{pasuram}' — use when the prabandham has decades.\n"
-                        "2. '{prabandham_code} {chapter}.{pasuram}' — use when the prabandham does not have decades.\n\n"
-                        "⚠️ Use this field ONLY when the user provides a specific prabandham and a relative verse number.\n"
-                        "Examples of valid usage:\n"
-                        "- User query: '3rd pasuram in the 8th Thiruvaimozhi of the 1st decade.'\n"
-                        "  → Convert to: '{prabandham_code} 1.8.3' and pass as `title` filter.\n"
-                        "- User query: '2nd pasuram of chapter 5 in [Prabandham with no decades].'\n"
-                        "  → Convert to: '{prabandham_code} 5.2' and pass as `title` filter.\n"
-                        "Do NOT use `title` for general queries or keyword searches — leave it empty in those cases."
-                    ),
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "verse",
-                    "label": "Absolute Pasuram Number",
-                    "datatype": "int",
-                    "is_unique": True,
-                    "description": (
-                        "Absolute verse number or pasuram number. Each verse has a unique number."
-                        # "Use it only when a specific prabandham name is NOT mentioned in the user query."
-                        "For e.g. 'Give me pasuram 1176'"
-                    ),
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "decade",
-                    "label": "Decade Number (Pathu)",
-                    "datatype": "int",
-                    "description": (
-                        "The decade (or `pathu` in Tamil) that this pasuram belongs to. decade is -1 when there is no associated decade."
-                    ),
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "chapter",
-                    "label": "Chapter Number",
-                    "datatype": "int",
-                    "description": (
-                        "chapter number of this pasuram. is -1 when there is no associated chapter number"
-                    ),
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "position_in_chapter",
-                    "label": "Relative Pasuram Number",
-                    "datatype": "int",
-                    "description": (
-                        "Relative verse number or pasuram number within a chapter."
-                        "Use it only when a specific prabandham name is mentioned in the user query."
-                        "For e.g. 'Give me the 5th pasuram from Thirupavai'"
-                    ),
-                    "show_as_filter": True,
-                },
-            ],
-            "pdf_path": "./data/divya_prabandham.pdf",
-            "source": "https://uveda.org",
-            "language": "tamil",
-            "example_labels": [
-                "About the five elements",
-                "About Garuda",
-                "Pasuram about Krishna's Flute",
-                "Andal's pasuram",
-                "Specific Pasuram (absolute)",
-                "Pasuram by Azhwar",
-                "Specific pasuram(relative)",
-                "Decade and Chapter Search",
-            ],
-            "examples": [
-                "five elements and their significance as defined in divya_prabandham",
-                "What is the significance of Garuda? Show some verses from divya prabandham that describe him.",
-                "Show me a pasuram that talks about how the animals and birds enjoy Krishna's flute playing.",
-                "Give me a pasuram by Andal",
-                "Show me Pasuram 1187 ",
-                "Show me a pasuram by Thondaradippodi azhwar",
-                "Give me the 2nd pasuram in the 3rd Thiruvaimozhi from the 2nd decade",
-                "Give me just a few words from the starting lines and reference links of all 11 pasurams from thiruvaimozhi 5th decade 4th chapter.",
-            ],
-            "llm_hints": [
-                "If the user wishes to query at a decade or chapter level for a given prabandham, use the direct metadata query on the appropriate fields once instead of querying the tool multiple times for each pasuram from the chapter."
-            ],
-        },
-        {
-            "name": "bhagavata_purana",
-            "title": "Bhagavatha Puranam",
-            "output_dir": "./output/bhagavata_purana",
-            "collection_name": "bhagavata_purana",
-            "unit": "page",
-            "metadata_fields": [
-                {
-                    "name": "file",
-                    "label": "File Name",
-                    "datatype": "str",
-                    "description": "name of the file from which the information was extracted",
-                },
-                {
-                    "name": "page",
-                    "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/bhagavata_purana.pdf",
-            "source": "https://dn790003.ca.archive.org/0/items/bhagavatapuranagitapress_201907/Bhagavata%20Purana%20-%20Gita%20Press_text.pdf",
-            "language": "san+eng",
-            "example_labels": ["Gajendra Moksham", "Prahalad"],
-            "examples": [
-                "State some verses that showcase the devotion of Gajendra the elephant",
-                "State some verses that showcase the devotion of Prahlada",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "kamba_ramayanam_en",
-            "title": "Kamba Ramayanam (English)",
-            "output_dir": "./output/kamba_ramayanam",
-            "collection_name": "kamba_ramayanam_en",
-            "unit": "verse",
-            "unit_field": "verse_number",
-            "field_mapping": {
-                "chapter_name": lambda doc: f"{doc.get('kandam','')}",
-                "title": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
-                "author": lambda doc: "Kamban",
-                "unit_index": "verse_number",
-                "verse": lambda doc: int(doc.get("verse_number", "0")),
-                "relative_path": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
-            },
-            "metadata_fields": [
-                {
-                    "name": "kandam",
-                    "label": "Kandam",
-                    "datatype": "str",
-                    "description": "The name of the Kandam or the chapter.",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_list_of_kandams(),
-                },
-                {
-                    "name": "padalam_en",
-                    "label": "Chapter Name",
-                    "datatype": "str",
-                    "description": "The name of the Padalam (Episode) in English.",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_list_of_padalams(),
-                },
-                {
-                    "name": "padalam_ta",
-                    "label": "Padalam name in Tamil",
-                    "datatype": "str",
-                    "description": "The name of the Padalam (Episode) in Tamil.",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_list_of_padalams_in_tamil(),
-                },
-                {
-                    "name": "page",
-                    "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "verse_number",
-                    "datatype": "int",
-                    "label": "Verse Number",
-                    "description": "Verse Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/kamba_ramayanam.pdf",
-            "source": "https://www.hindupedia.com/images/1/13/Kamba_Ramayanam_I.pdf",
-            "language": "tamil",
-            "example_labels": [
-                "About Jatayu",
-                "About Hanuman",
-                "About Vali",
-                "About Sita",
-                "About Ravana",
-            ],
-            "examples": [
-                "What is the significance of Jatayu? show some sanskrit verses to support the argument",
-                "Show some verses where Hanuman is mentioned",
-                "How did Rama kill Vali",
-                "How was Sita abducted",
-                "How did Rama kill Ravana?",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "kamba_ramayanam",
-            "title": "Kamba Ramayanam (Tamil)",
-            "output_dir": "./output/kamba_ramayanam",
-            "collection_name": "kamba_ramayanam",
-            "unit": "chunk",
-            "unit_field": "chunk_index",
-            "field_mapping": {
-                "chapter_name": lambda doc: f"{doc.get('kandam','')}",
-                "title": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
-                "author": lambda doc: "Kamban",
-                "unit_index": "chunk_index",
-                "verse": lambda doc: int(doc.get("verse_number", "0")),
-                "relative_path": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
-            },
-            "metadata_fields": [
-                {
-                    "name": "chunk_index",
-                    "label": "Page Index",
-                    "datatype": "int",
-                    "description": "The index of the chunk",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "filename",
-                    "label": "File name from the source",
-                    "datatype": "str",
-                    "description": "The name of the file.",
-                },
-            ],
-            "pdf_path": "./data/kamba_ramayanam.pdf",
-            "source": "https://archive.org/details/vrajeshkumar_gmail_061/01-%E0%AE%AA%E0%AE%BE%E0%AE%B2%20%E0%AE%95%E0%AE%BE%E0%AE%A3%E0%AF%8D%E0%AE%9F%E0%AE%AE%E0%AF%8D/page/n15/mode/2up",
-            "language": "tamil",
-            "example_labels": [
-                "About Jatayu",
-                "About Hanuman",
-                "About Vali",
-                "About Sita",
-                "About Ravana",
-            ],
-            "examples": [
-                "What is the significance of Jatayu? show some sanskrit verses to support the argument",
-                "Show some verses where Hanuman is mentioned",
-                "How did Rama kill Vali",
-                "How was Sita abducted",
-                "How did Rama kill Ravana?",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "chathusloki",
-            "title": "Chathusloki by Sri Alavandar",
-            "output_dir": "./output/chathusloki",
-            "collection_name": "chathusloki",
-            "unit": "verse",
-            "unit_field": "sloka_number",
-            "field_mapping": {
-                "text": "sloka_devanagari",
-                "title": lambda doc: f"Verse {doc.get('verse','')}",
-                "unit_index": "sloka_number",
-                "transliteration": "sloka_english_transliteration",
-                "word_by_word_native": "meaning",
-                "author": lambda doc: f"Sri Aalavandhaar",
-                "relative_path": lambda doc: f"Chathusloki-{doc.get("sloka_number","")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "sloka_number",
-                    "label": "Slokam Number",
-                    "datatype": "int",
-                    "description": "The index of the sloka or verse",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "meaning_short",
-                    "label": "Short meaning",
-                    "datatype": "str",
-                    "description": "A short meaning of the sanskrit verse in English.",
-                    "show_as_filter": True,
-                },
-            ],
-            "pdf_path": "./data/chathusloki.pdf",
-            "source": "https://www.sadagopan.org/ebook/pdf/Chatusloki%20-%20VS.pdf",
-            "language": "san+eng",
-            "example_labels": ["Recite a sloka", "Commentary", "Role of Sridevi"],
-            "examples": [
-                "Recite the 1st sloka from Chathusloki",
-                "Show detailed commentary for sloka 2 from Chathusloki",
-                "What is the role of Sri Devi in the universe according to the Chathusloki?",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "sri_stavam",
-            "title": "Sri Stavam by Sri Koorathazhwan",
-            "output_dir": "./output/sri_stavam",
-            "collection_name": "sri_stavam",
-            "unit": "slokam",
-            "unit_field": "sloka_number",
-            "field_mapping": {
-                "text": "sanskrit",
-                "title": lambda doc: f"Verse {doc.get('verse','')}",
-                "unit_index": "sloka_number",
-                "transliteration": "transliteration",
-                "word_by_word_native": "meaning_short",
-                "author": lambda doc: f"Sri Koorathazhwan",
-                "relative_path": lambda doc: f"Sri Stavam-{doc.get("sloka_number","")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "sloka_number",
-                    "label": "Slokam Number",
-                    "datatype": "int",
-                    "description": "The index of the sloka or verse",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "meaning_short",
-                    "label": "Short meaning",
-                    "datatype": "str",
-                    "description": "A short meaning of the sanskrit verse in English.",
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "Verse in sanskrit",
-                },
-                {
-                    "name": "transliteration",
-                    "label": "English Transliteration",
-                    "datatype": "str",
-                    "description": "Verse transliterated to English",
-                    "show_as_filter": True,
-                },
-            ],
-            "pdf_path": "./data/sri_stavam.pdf",
-            "source": "https://www.sadagopan.org/ebook/pdf/Sri%20Stavam.pdf",
-            "language": "san+eng",
-            "example_labels": ["Recite a sloka", "Commentary", "Role of Sridevi"],
-            "examples": [
-                "Recite the 1st sloka from Sri Stavam",
-                "Show detailed commentary for sloka 2 from Sri Stavam",
-                "What is the role of Sri Devi in the universe according to the Sri Stavam?",
-            ],
-            "llm_hints": [
-                "if the user asks for nth sloka, do a metadata search on the `verse` field."
-            ],
-        },
-        {
-            "name": "yt_metadata",
-            "title": "Sampradayam in YouTube",
-            "output_dir": "./output/yt_metadata",
-            "collection_name": "yt_metadata",
-            "collection_embedding_fn": "openai",
-            "unit": "video",
-            "field_mapping": {
-                "text": "description",
-                "title": "video_title",
-                "author": "channel_title",
-                "reference_link": lambda doc: f"https://www.youtube.com/watch?v={doc.get('video_id','')}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "video_id",
-                    "label": "Video ID",
-                    "datatype": "str",
-                    "description": "The video id as in YouTube",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "video_title",
-                    "label": "Video Title",
-                    "datatype": "str",
-                    "description": "The title of the video as in YouTube",
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "description",
-                    "label": "Video Description",
-                    "datatype": "str",
-                    "description": "Description as in YouTube",
-                    "show_as_filter": True,
-                },
-                {
-                    "name": "channel_url",
-                    "label": "Channel URL",
-                    "datatype": "str",
-                    "description": "URL of the YouTube Channel",
-                },
-                {
-                    "name": "channel_title",
-                    "label": "Channel Title",
-                    "datatype": "str",
-                    "description": "Title of the YouTube Channel",
-                    "show_as_filter": True,
-                },
-            ],
-            "pdf_path": "./data/none.pdf",
-            "source": "https://youtube.com",
-            "language": "san+eng+tam",
-            "example_labels": ["Srirangam", "Pasuram video"],
-            "examples": [
-                "Show me YouTube videos that talk about Srirangam",
-                "Show me lyrics of 1st pasuram of 1st decade in the 4st Thiruvaimozhi. Also show the related youtube videos.",
-            ],
-            "llm_hints": [
-                "if the user asks for YouTube videos, DO NOT do a web search, instead do a search on this collection."
-            ],
-        },
-        {
-            "name": "pancha_sooktham",
-            "title": "Pancha Sooktham",
-            "output_dir": "./output/pancha_sooktham",
-            "collection_name": "pancha_sooktham",
-            "collection_embedding_fn": "openai",
-            "unit": "verse",
-            "unit_field": "relative_verse_number",
-            "chapter_order": lambda: get_chapter_order_from_pancha_sooktham(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "unit_index": "relative_verse_number",
-                "transliteration": lambda doc: f"English: \n{doc.get('english')}\n\nதமிழ்: \n{doc.get('tamil')}\n\nతెలుగు: \n{doc.get('telugu')}\n\nಕನ್ನಡ: \n{doc.get('kannada')}",
-                "chapter_name": "chapter_title",
-                "relative_path": lambda doc: f"{doc.get('chapter_title',"")}-{doc.get('relative_verse_number',"")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "chapter_title",
-                    "datatype": "str",
-                    "label": "Chapter Name",
-                    "description": "Name of the Chapter",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_pancha_sooktham_chapters(),
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "relative_verse_number",
-                    "label": "Relative Verse Number",
-                    "datatype": "int",
-                    "description": "The relative verse number of the sloka inside the chapter.",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/pancha_sooktham.pdf",
-            "source": "https://stotranidhi.com/",
-            "language": "san+eng",
-            "example_labels": [
-                "Purusha Sooktham",
-            ],
-            "examples": [
-                "Show some verses from purusha sooktham",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "taitriya_upanishad",
-            "title": "Taitriya Upanishad",
-            "output_dir": "./output/taitriya_upanishad",
-            "collection_name": "taitriya_upanishad",
-            "collection_embedding_fn": "openai",
-            "unit": "verse",
-            "unit_field": "relative_verse_number",
-            "chapter_order": lambda: get_chapter_order_from_taitriya_upanishad(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "unit_index": "relative_verse_number",
-                "transliteration": lambda doc: f"English: \n{doc.get('english')}\n\nதமிழ்: \n{doc.get('tamil')}\n\nతెలుగు: \n{doc.get('telugu')}\n\nಕನ್ನಡ: \n{doc.get('kannada')}",
-                "chapter_name": "chapter_title",
-                "relative_path": lambda doc: f"{doc.get('chapter_title',"")}-{doc.get('relative_verse_number',"")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "chapter_title",
-                    "datatype": "str",
-                    "label": "Chapter Name",
-                    "description": "Name of the Chapter",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_taitriya_upanishad_chapters(),
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "relative_verse_number",
-                    "label": "Relative Verse Number",
-                    "datatype": "int",
-                    "description": "The relative verse number of the sloka inside the chapter.",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/taitriya_upanishad.pdf",
-            "source": "https://stotranidhi.com/",
-            "language": "san+eng",
-            "example_labels": [
-                "Seeksha Valli",
-            ],
-            "examples": [
-                "Show some verses from Seeksha Valli",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "shanthi_panchakam",
-            "title": "Shanthi Panchakam",
-            "output_dir": "./output/shanthi_panchakam",
-            "collection_name": "shanthi_panchakam",
-            "collection_embedding_fn": "openai",
-            "unit": "verse",
-            "unit_field": "relative_verse_number",
-            "chapter_order": lambda: get_chapter_order_from_shanthi_panchakam(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "unit_index": "relative_verse_number",
-                "transliteration": lambda doc: f"English: \n{doc.get('english')}\n\nதமிழ்: \n{doc.get('tamil')}\n\nతెలుగు: \n{doc.get('telugu')}\n\nಕನ್ನಡ: \n{doc.get('kannada')}",
-                "chapter_name": "chapter_title",
-                "relative_path": lambda doc: f"{doc.get('chapter_title',"")}-{doc.get('relative_verse_number',"")}",
-            },
-            "metadata_fields": [
-                {
-                    "name": "chapter_title",
-                    "datatype": "str",
-                    "label": "Chapter Name",
-                    "description": "Name of the Chapter",
-                    "show_as_filter": True,
-                    "component": "dropdown",
-                    "lov": lambda: get_shanthi_panchakam_chapters(),
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "relative_verse_number",
-                    "label": "Relative Verse Number",
-                    "datatype": "int",
-                    "description": "The relative verse number of the sloka inside the chapter.",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-            ],
-            "pdf_path": "./data/shanthi_panchakam.pdf",
-            "source": "https://vignanam.org/english/shanti-panchakam.html",
-            "language": "san+eng",
-            "example_labels": [
-                "Shanthi Panchakam",
-            ],
-            "examples": [
-                "Show some verses from Shanthi Panchakam",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "taitriya_samhitha",
-            "title": "Taitriya Samhitha",
-            "output_dir": "./output/taitriya_samhitha",
-            "collection_name": "taitriya_samhitha",
-            "collection_embedding_fn": "openai",
-            "unit": "panchadhi",
-            "unit_field": "panchadhi",
-            # "chapter_order": lambda: get_chapter_order_from_taitriya_samhitha(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "unit_index": "panchadhi",
-                "transliteration": "transliteration",
-                "chapter_name": lambda doc: (
-                    doc.get("prashnam_name")
-                    if doc.get("prashnam_name") and doc.get("prashnam_name") != "-"
-                    else (
-                        doc.get("kandam_name")
-                        if doc.get("kandam_name") and doc.get("kandam_name") != "-"
-                        else "-"
-                    )
-                ),
-                "relative_path": lambda doc: (
-                    "-"
-                    if not doc.get("panchadhi") or doc.get("panchadhi") == "-"
-                    else f"{doc.get('kandam')}.{doc.get('prashnam')}.{doc.get('anuvakam')}.{doc.get('panchadhi_index')} || {doc.get('panchadhi_sa')} ||"
-                ),
-            },
-            "metadata_fields": [
-                {
-                    "name": "kandam",
-                    "datatype": "int",
-                    "label": "Kandam Number",
-                    "description": "Kandam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "prashnam",
-                    "datatype": "int",
-                    "label": "Prashnam Number",
-                    "description": "Prashnam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "anuvakam",
-                    "datatype": "int",
-                    "label": "Anuvakam Number",
-                    "description": "Anuvakam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "panchadhi_index",
-                    "datatype": "int",
-                    "label": "Relative Panchadhi Number",
-                    "description": "Relative Panchadhi Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "panchadhi",
-                    "datatype": "int",
-                    "label": "Absolute Panchadhi Number",
-                    "description": "Absolute Panchadhi Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "transliteration",
-                    "label": "Transliteration in english",
-                    "datatype": "str",
-                    "description": "The original sloka transliterated in English.",
-                },
-            ],
-            "pdf_path": "./data/taitriya_samhitha.pdf",
-            "source": "https://sanskritdocuments.org/doc_veda/taittirIyasamhitA.html",
-            "language": "san+eng",
-            "example_labels": [
-                "Taitriya Samhitha",
-            ],
-            "examples": [
-                "Show some verses from Taitriya Samhitha",
-            ],
-            "llm_hints": [],
-        },
-        {
-            "name": "taitriya_brahmanam",
-            "title": "Taitriya Brahmanam",
-            "output_dir": "./output/taitriya_brahmanam",
-            "collection_name": "taitriya_brahmanam",
-            "collection_embedding_fn": "openai",
-            "unit": "panchadhi",
-            "unit_field": "panchadhi",
-            # "chapter_order": lambda: get_chapter_order_from_taitriya_brahmanam(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "unit_index": "panchadhi",
-                "transliteration": "transliteration",
-                "chapter_name": lambda doc: (
-                    doc.get("prapaatakam_name")
-                    if doc.get("prapaatakam_name") and doc.get("prapaatakam_name") != "-"
-                    else (
-                        doc.get("ashtakam_name")
-                        if doc.get("ashtakam_name") and doc.get("ashtakam_name") != "-"
-                        else "-"
-                    )
-                ),
-                "relative_path": lambda doc: (
-                    "-"
-                    if not doc.get("panchadhi") or doc.get("panchadhi") == "-"
-                    else f"{doc.get('ashtakam')}.{doc.get('prapaatakam')}.{doc.get('anuvakam')}.{doc.get('panchadhi_index')} || {doc.get('panchadhi_sa')} ||"
-                ),
-            },
-            "metadata_fields": [
-                {
-                    "name": "ashtakam",
-                    "datatype": "int",
-                    "label": "Ashtakam Number",
-                    "description": "Ashtakam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "prapaatakam",
-                    "datatype": "int",
-                    "label": "Prapaatakam Number",
-                    "description": "Prapaatakam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "anuvakam",
-                    "datatype": "int",
-                    "label": "Anuvakam Number",
-                    "description": "Anuvakam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "panchadhi_index",
-                    "datatype": "int",
-                    "label": "Relative Panchadhi Number",
-                    "description": "Relative Panchadhi Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "panchadhi",
-                    "datatype": "int",
-                    "label": "Absolute Panchadhi Number",
-                    "description": "Absolute Panchadhi Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "transliteration",
-                    "label": "Transliteration in english",
-                    "datatype": "str",
-                    "description": "The original sloka transliterated in English.",
-                },
-            ],
-            "pdf_path": "./data/taitriya_brahmanam.pdf",
-            "source": "https://sanskritdocuments.org/doc_veda/taittirIyabrAhmaNam.html",
-            "language": "san+eng",
-            "example_labels": [
-                "Taitriya Brahmanam",
-            ],
-            "examples": [
-                "Show some verses from Taitriya Brahmanam",
-            ],
-            "llm_hints": [],
-        },        
-        {
-            "name": "katakam",
-            "title": "Katakam",
-            "output_dir": "./output/katakam",
-            "collection_name": "katakam",
-            "collection_embedding_fn": "openai",
-            "unit": "panchadhi",
-            "unit_field": "panchadhi",
-            # "chapter_order": lambda: get_chapter_order_from_katakam(),
-            "field_mapping": {
-                "text": "sanskrit",
-                "unit_index": "panchadhi",
-                "transliteration": "transliteration",
-                "chapter_name": lambda doc: (
-                    doc.get("prapaatakam_name")
-                    if doc.get("prapaatakam_name") and doc.get("prapaatakam_name") != "-"
-                    else "॥ काठकम् ॥"
-                ),
-                "relative_path": lambda doc: (
-                    "-"
-                    if not doc.get("panchadhi") or doc.get("panchadhi") == "-"
-                    else f"{doc.get('prapaatakam')}.{doc.get('anuvakam')}.{doc.get('panchadhi_index')} || {doc.get('panchadhi_sa')} ||"
-                ),
-            },
-            "metadata_fields": [
-                {
-                    "name": "prapaatakam",
-                    "datatype": "int",
-                    "label": "Prapaatakam Number",
-                    "description": "Prapaatakam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "anuvakam",
-                    "datatype": "int",
-                    "label": "Anuvakam Number",
-                    "description": "Anuvakam Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "panchadhi_index",
-                    "datatype": "int",
-                    "label": "Relative Panchadhi Number",
-                    "description": "Relative Panchadhi Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "panchadhi",
-                    "datatype": "int",
-                    "label": "Absolute Panchadhi Number",
-                    "description": "Absolute Panchadhi Number",
-                    "show_as_filter": True,
-                    "is_unique": True,
-                },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
-                {
-                    "name": "transliteration",
-                    "label": "Transliteration in english",
-                    "datatype": "str",
-                    "description": "The original sloka transliterated in English.",
-                },
-            ],
-            "pdf_path": "./data/katakam.pdf",
-            "source": "https://sanskritdocuments.org/doc_veda/taittirIyabrAhmaNam.html",
-            "language": "san+eng",
-            "example_labels": [
-                "Katakam",
-            ],
-            "examples": [
-                "Show some verses from Katakam",
-            ],
-            "llm_hints": [],
-        },        
-    ]
-
+    scriptures = scripture_configurations
     def get_scripture_by_collection(self, collection_name: str):
         return [
             scripture
@@ -1365,3 +145,7 @@ class SanatanConfig:
         )
         collection_name = config.get("collection_name")
         return collection_name
+
+if __name__ == "__main__":
+    print(SanatanConfig.scriptures)
+    [scripture["collection_name"] for scripture in SanatanConfig.scriptures]

data/dropbox/scriptures/divya_prabandham/video_metadata.json

@@ -77,30 +77,126 @@
         "video_url": "https://www.youtube.com/watch?v=6DFMA-hG7Vs",
         "type": "virutham"
     },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 474,
+        "video_url": "https://www.youtube.com/shorts/E3wanN1FiCs",
+        "type": "upanyasam"
+    },
     {
         "scripture": "divya_prabandham",
         "global_index": 475,
         "video_url": "https://www.youtube.com/watch?v=hBR9Cu0sev0",
         "type": "virutham"
     },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 475,
+        "video_url": "https://www.youtube.com/shorts/lF3NMduim2I",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 475,
+        "video_url": "https://www.youtube.com/shorts/rcYDiw4q2Po",
+        "type": "upanyasam 2"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 475,
+        "video_url": "https://www.youtube.com/shorts/fHFBUMkcd_o",
+        "type": "upanyasam 3"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 475,
+        "video_url": "https://www.youtube.com/shorts/Flyra2mqopM",
+        "type": "upanyasam 4"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 475,
+        "video_url": "https://www.youtube.com/shorts/Px6VoeaVJu4",
+        "type": "upanyasam 4"
+    },
     {
         "scripture": "divya_prabandham",
         "global_index": 476,
         "video_url": "https://www.youtube.com/watch?v=gEx7wtE7kxc",
         "type": "virutham"
     },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 476,
+        "video_url": "https://www.youtube.com/shorts/ccyBfd3vQcU",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 476,
+        "video_url": "https://www.youtube.com/shorts/y0N65owAa5c",
+        "type": "upanyasam 2"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 476,
+        "video_url": "https://www.youtube.com/shorts/T3GH0YOpHc0",
+        "type": "upanyasam 3"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 476,
+        "video_url": "https://www.youtube.com/shorts/5idsyjWrpVg",
+        "type": "upanyasam 4"
+    },
     {
         "scripture": "divya_prabandham",
         "global_index": 477,
         "video_url": "https://www.youtube.com/watch?v=wE2C-Xzrcfg",
         "type": "virutham"
     },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 477,
+        "video_url": "https://www.youtube.com/shorts/N8Z042bZdTQ",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 477,
+        "video_url": "https://www.youtube.com/shorts/EJyqA1QhsV4",
+        "type": "upanyasam 2"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 477,
+        "video_url": "https://www.youtube.com/shorts/ksEh6PQ1Jh4",
+        "type": "upanyasam 3"
+    },
     {
         "scripture": "divya_prabandham",
         "global_index": 478,
         "video_url": "https://www.youtube.com/watch?v=ql3Be2uckjY",
         "type": "virutham"
     },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 478,
+        "video_url": "https://www.youtube.com/shorts/dLguVgyXVKA",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 478,
+        "video_url": "https://www.youtube.com/shorts/FE-mI732zmQ",
+        "type": "upanyasam 2"
+    },
+    {
+        "scripture": "divya_prabandham",
+        "global_index": 478,
+        "video_url": "https://www.youtube.com/shorts/lb5IxMn2M_8",
+        "type": "upanyasam 3"
+    },
     {
         "scripture": "divya_prabandham",
         "global_index": 479,

modules/config/__init__.py

@@ -0,0 +1,40 @@
+from modules.config.bhagavat_gita import bhagavat_gita_config
+from modules.config.bhagavata_purana import bhagavata_purana_config
+from modules.config.chathusloki import chathusloki_config
+from modules.config.divya_prabandham import divya_prabandham_config
+from modules.config.kamba_ramayanam import kamba_ramayanam_config
+from modules.config.kamba_ramayanam_en import kamba_ramayanam_en_config
+from modules.config.katakam import katakam_config
+from modules.config.pancha_sooktham import pancha_sooktham_config
+from modules.config.shanthi_panchakam import shanthi_panchakam_config
+from modules.config.shukla_yajur_vedam import shukla_yajur_vedam_config
+from modules.config.sri_stavam import sri_stavam_config
+from modules.config.taitriya_brahmanam import taitriya_brahmanam_config
+from modules.config.taitriya_samhitha import taitriya_samhitha_config
+from modules.config.taitriya_upanishad import taitriya_upanishad_config
+from modules.config.valmiki_ramayanam import valmiki_ramayanam_config
+from modules.config.vishnu_puranam import vishnu_puranam_config
+from modules.config.vishnu_sahasranamam import vishnu_sahasranamam_config
+from modules.config.yt_metadata import yt_metadata_config
+
+
+scripture_configurations = [
+    bhagavat_gita_config,
+    bhagavata_purana_config,
+    chathusloki_config,
+    divya_prabandham_config,
+    kamba_ramayanam_en_config,
+    kamba_ramayanam_config,
+    katakam_config,
+    pancha_sooktham_config,
+    shanthi_panchakam_config,
+    shukla_yajur_vedam_config,
+    sri_stavam_config,
+    taitriya_brahmanam_config,
+    taitriya_samhitha_config,
+    taitriya_upanishad_config,
+    valmiki_ramayanam_config,
+    vishnu_puranam_config,
+    vishnu_sahasranamam_config,
+    yt_metadata_config,
+]

modules/config/bhagavat_gita.py

@@ -0,0 +1,85 @@
+from modules.scripture_helpers.bhagavat_gita_helper import (
+    get_bhagavat_gita_chapters,
+    get_chapter_order_from_bhagavat_gita,
+)
+
+bhagavat_gita_config = {
+    "name": "bhagavat_gita",
+    "title": "Bhagavat Gita",
+    "output_dir": "./output/bhagavat_gita",
+    "collection_name": "bhagavat_gita_openai",
+    "collection_embedding_fn": "openai",
+    "unit": "verse",
+    "unit_field": "relative_verse_number",
+    "chapter_order": lambda: get_chapter_order_from_bhagavat_gita(),
+    "audio_storage": "external",
+    "field_mapping": {
+        "text": "sanskrit",
+        "title": "verse_title",
+        "unit_index": "relative_verse_number",
+        "transliteration": "transliteration",
+        "word_by_word_native": "word_by_word_meaning",
+        "translation": "translation",
+        "author": lambda doc: f"Maharishi Vedvyas Ji",
+        "chapter_name": "chapter_title",
+        "relative_path": lambda doc: f"{doc.get("chapter_title","")}-{doc.get("relative_verse_number","")}",
+    },
+    "metadata_fields": [
+        {
+            "name": "chapter_title",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Name of the Chapter",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_bhagavat_gita_chapters(),
+            "is_unique": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "translation",
+            "label": "English Translation",
+            "datatype": "str",
+            "description": "The english translation.",
+        },
+        {
+            "name": "transliteration",
+            "label": "English Transliteration",
+            "datatype": "str",
+            "description": "The english transliteration.",
+        },
+        {
+            "name": "relative_verse_number",
+            "label": "Relative Verse Number",
+            "datatype": "int",
+            "description": "The relative verse number of the sloka inside the chapter.",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/bhagavat_gita.pdf",
+    "source": "https://vivekavani.com/",
+    "language": "san+eng",
+    "example_labels": [
+        "About Arjuna",
+        "About Karma",
+        "About birth and death",
+        "About the battle field",
+        "About Krishna's form",
+        "Krishna's Teachings",
+    ],
+    "examples": [
+        "Show some verses where Krishna advises Arjuna",
+        "What does Krishna say about Karma",
+        "What does Krishna say about birth and death",
+        "describe the battle field",
+        "How did Arjuna respond upon witnessing Krishna’s Vishwarupa?"
+        "What teachings did Krishna share in the Gita?",
+    ],
+    "llm_hints": [],
+}

modules/config/bhagavata_purana.py

@@ -0,0 +1,32 @@
+bhagavata_purana_config = {
+    "name": "bhagavata_purana",
+    "title": "Bhagavatha Puranam",
+    "output_dir": "./output/bhagavata_purana",
+    "collection_name": "bhagavata_purana",
+    "unit": "page",
+    "metadata_fields": [
+        {
+            "name": "file",
+            "label": "File Name",
+            "datatype": "str",
+            "description": "name of the file from which the information was extracted",
+        },
+        {
+            "name": "page",
+            "datatype": "int",
+            "label": "Page Number",
+            "description": "Page number from the source",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/bhagavata_purana.pdf",
+    "source": "https://dn790003.ca.archive.org/0/items/bhagavatapuranagitapress_201907/Bhagavata%20Purana%20-%20Gita%20Press_text.pdf",
+    "language": "san+eng",
+    "example_labels": ["Gajendra Moksham", "Prahalad"],
+    "examples": [
+        "State some verses that showcase the devotion of Gajendra the elephant",
+        "State some verses that showcase the devotion of Prahlada",
+    ],
+    "llm_hints": [],
+}

modules/config/chathusloki.py

@@ -0,0 +1,44 @@
+chathusloki_config = {
+    "name": "chathusloki",
+    "title": "Chathusloki by Sri Alavandar",
+    "output_dir": "./output/chathusloki",
+    "collection_name": "chathusloki",
+    "unit": "verse",
+    "unit_field": "sloka_number",
+    "field_mapping": {
+        "text": "sloka_devanagari",
+        "title": lambda doc: f"Verse {doc.get('verse','')}",
+        "unit_index": "sloka_number",
+        "transliteration": "sloka_english_transliteration",
+        "word_by_word_native": "meaning",
+        "author": lambda doc: f"Sri Aalavandhaar",
+        "relative_path": lambda doc: f"Chathusloki-{doc.get("sloka_number","")}",
+    },
+    "metadata_fields": [
+        {
+            "name": "sloka_number",
+            "label": "Slokam Number",
+            "datatype": "int",
+            "description": "The index of the sloka or verse",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "meaning_short",
+            "label": "Short meaning",
+            "datatype": "str",
+            "description": "A short meaning of the sanskrit verse in English.",
+            "show_as_filter": True,
+        },
+    ],
+    "pdf_path": "./data/chathusloki.pdf",
+    "source": "https://www.sadagopan.org/ebook/pdf/Chatusloki%20-%20VS.pdf",
+    "language": "san+eng",
+    "example_labels": ["Recite a sloka", "Commentary", "Role of Sridevi"],
+    "examples": [
+        "Recite the 1st sloka from Chathusloki",
+        "Show detailed commentary for sloka 2 from Chathusloki",
+        "What is the role of Sri Devi in the universe according to the Chathusloki?",
+    ],
+    "llm_hints": [],
+}

modules/config/divya_prabandham.py

@@ -0,0 +1,165 @@
+import nalayiram_helper
+
+divya_prabandham_config = {
+    "name": "divya_prabandham",
+    "title": "4000 Divya Prabandham",
+    "output_dir": "./output/divya_prabandham",
+    "collection_name": "divya_prabandham",
+    "collection_embedding_fn": "openai",
+    "unit": "pasuram",
+    "unit_field": "verse",
+    "credits": "We owe deep gratitude to uveda.org for providing such beautifully curated data. 🙏 This section would not have been possible without the dedicated efforts of the Uveda team.",
+    "field_mapping": {
+        "text": "pasuram_ta",
+        "title": lambda doc: f"{doc.get('prabandham_name','')} {doc.get('chapter','')}-{doc.get('decade','')}:{doc.get('position_in_chapter','')}",
+        "location": "divya_desams",
+        "word_by_word_native": "wbw_ta",
+        "unit_index": "verse",
+        "transliteration": "pasuram_en",
+        "reference_link": "html_url",
+        "author": "azhwar_name",
+        "chapter_name": "prabandham_name",
+        "relative_path": lambda doc: "-".join(
+            filter(
+                None,
+                [
+                    doc.get("prabandham_name", ""),
+                    *(
+                        str(doc.get(k))
+                        for k in ["decade", "chapter", "position_in_chapter"]
+                        if doc.get(k, -1) != -1
+                    ),
+                ],
+            )
+        ),
+    },
+    "metadata_fields": [
+        {
+            "name": "prabandham_code",
+            "label": "Prabandham Code",
+            "datatype": "str",
+            "description": "contains the short prabandham_code. e.g. `TPL` for `Thiruppallandu`",
+        },
+        {
+            "name": "prabandham_name",
+            "label": "Prabandham Name",
+            "datatype": "str",
+            "description": "contains the prabandham name. e.g. `Thiruppallandu`",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: [
+                p.prabandham_name
+                for p in nalayiram_helper.get_standardized_prabandham_names()
+            ],
+        },
+        {
+            "name": "azhwar_name",
+            "label": "Azhwar Name",
+            "datatype": "str",
+            "description": "contains the azhwar name. e.g. `Thirumangai Azhwar`",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: [
+                azhwar_name
+                for azhwar_name in nalayiram_helper.get_standardized_azhwar_names()
+            ],
+        },
+        {
+            "name": "divya_desams",
+            "label": "Divya Desams",
+            "datatype": "str",
+            "description": "comma separated list of divya desams. e.g. Thiruneermalai,Thiruvallikkeni.",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: [
+                divya_desam_name
+                for divya_desam_name in nalayiram_helper.get_standardized_divya_desam_names()
+            ],
+        },
+        {
+            "name": "title",
+            "label": "Pasuram Title",
+            "datatype": "str",
+            "description": (
+                "Exact title of a pasuram in one of the following formats:\n"
+                "1. '{prabandham_code} {decade}.{chapter}.{pasuram}' — use when the prabandham has decades.\n"
+                "2. '{prabandham_code} {chapter}.{pasuram}' — use when the prabandham does not have decades.\n\n"
+                "⚠️ Use this field ONLY when the user provides a specific prabandham and a relative verse number.\n"
+                "Examples of valid usage:\n"
+                "- User query: '3rd pasuram in the 8th Thiruvaimozhi of the 1st decade.'\n"
+                "  → Convert to: '{prabandham_code} 1.8.3' and pass as `title` filter.\n"
+                "- User query: '2nd pasuram of chapter 5 in [Prabandham with no decades].'\n"
+                "  → Convert to: '{prabandham_code} 5.2' and pass as `title` filter.\n"
+                "Do NOT use `title` for general queries or keyword searches — leave it empty in those cases."
+            ),
+            "show_as_filter": True,
+        },
+        {
+            "name": "verse",
+            "label": "Absolute Pasuram Number",
+            "datatype": "int",
+            "is_unique": True,
+            "description": (
+                "Absolute verse number or pasuram number. Each verse has a unique number."
+                # "Use it only when a specific prabandham name is NOT mentioned in the user query."
+                "For e.g. 'Give me pasuram 1176'"
+            ),
+            "show_as_filter": True,
+        },
+        {
+            "name": "decade",
+            "label": "Decade Number (Pathu)",
+            "datatype": "int",
+            "description": (
+                "The decade (or `pathu` in Tamil) that this pasuram belongs to. decade is -1 when there is no associated decade."
+            ),
+            "show_as_filter": True,
+        },
+        {
+            "name": "chapter",
+            "label": "Chapter Number",
+            "datatype": "int",
+            "description": (
+                "chapter number of this pasuram. is -1 when there is no associated chapter number"
+            ),
+            "show_as_filter": True,
+        },
+        {
+            "name": "position_in_chapter",
+            "label": "Relative Pasuram Number",
+            "datatype": "int",
+            "description": (
+                "Relative verse number or pasuram number within a chapter."
+                "Use it only when a specific prabandham name is mentioned in the user query."
+                "For e.g. 'Give me the 5th pasuram from Thirupavai'"
+            ),
+            "show_as_filter": True,
+        },
+    ],
+    "pdf_path": "./data/divya_prabandham.pdf",
+    "source": "https://uveda.org",
+    "language": "tamil",
+    "example_labels": [
+        "About the five elements",
+        "About Garuda",
+        "Pasuram about Krishna's Flute",
+        "Andal's pasuram",
+        "Specific Pasuram (absolute)",
+        "Pasuram by Azhwar",
+        "Specific pasuram(relative)",
+        "Decade and Chapter Search",
+    ],
+    "examples": [
+        "five elements and their significance as defined in divya_prabandham",
+        "What is the significance of Garuda? Show some verses from divya prabandham that describe him.",
+        "Show me a pasuram that talks about how the animals and birds enjoy Krishna's flute playing.",
+        "Give me a pasuram by Andal",
+        "Show me Pasuram 1187 ",
+        "Show me a pasuram by Thondaradippodi azhwar",
+        "Give me the 2nd pasuram in the 3rd Thiruvaimozhi from the 2nd decade",
+        "Give me just a few words from the starting lines and reference links of all 11 pasurams from thiruvaimozhi 5th decade 4th chapter.",
+    ],
+    "llm_hints": [
+        "If the user wishes to query at a decade or chapter level for a given prabandham, use the direct metadata query on the appropriate fields once instead of querying the tool multiple times for each pasuram from the chapter."
+    ],
+}

modules/config/kamba_ramayanam.py

@@ -0,0 +1,50 @@
+kamba_ramayanam_config = {
+    "name": "kamba_ramayanam",
+    "title": "Kamba Ramayanam (Tamil)",
+    "output_dir": "./output/kamba_ramayanam",
+    "collection_name": "kamba_ramayanam",
+    "unit": "chunk",
+    "unit_field": "chunk_index",
+    "field_mapping": {
+        "chapter_name": lambda doc: f"{doc.get('kandam','')}",
+        "title": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
+        "author": lambda doc: "Kamban",
+        "unit_index": "chunk_index",
+        "verse": lambda doc: int(doc.get("verse_number", "0")),
+        "relative_path": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
+    },
+    "metadata_fields": [
+        {
+            "name": "chunk_index",
+            "label": "Page Index",
+            "datatype": "int",
+            "description": "The index of the chunk",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "filename",
+            "label": "File name from the source",
+            "datatype": "str",
+            "description": "The name of the file.",
+        },
+    ],
+    "pdf_path": "./data/kamba_ramayanam.pdf",
+    "source": "https://archive.org/details/vrajeshkumar_gmail_061/01-%E0%AE%AA%E0%AE%BE%E0%AE%B2%20%E0%AE%95%E0%AE%BE%E0%AE%A3%E0%AF%8D%E0%AE%9F%E0%AE%AE%E0%AF%8D/page/n15/mode/2up",
+    "language": "tamil",
+    "example_labels": [
+        "About Jatayu",
+        "About Hanuman",
+        "About Vali",
+        "About Sita",
+        "About Ravana",
+    ],
+    "examples": [
+        "What is the significance of Jatayu? show some sanskrit verses to support the argument",
+        "Show some verses where Hanuman is mentioned",
+        "How did Rama kill Vali",
+        "How was Sita abducted",
+        "How did Rama kill Ravana?",
+    ],
+    "llm_hints": [],
+}

modules/config/kamba_ramayanam_en.py

@@ -0,0 +1,84 @@
+from modules.scripture_helpers.kamba_ramayanam_helper import (
+    get_list_of_kandams,
+    get_list_of_padalams,
+    get_list_of_padalams_in_tamil,
+)
+
+kamba_ramayanam_en_config = {
+    "name": "kamba_ramayanam_en",
+    "title": "Kamba Ramayanam (English)",
+    "output_dir": "./output/kamba_ramayanam",
+    "collection_name": "kamba_ramayanam_en",
+    "unit": "verse",
+    "unit_field": "verse_number",
+    "field_mapping": {
+        "chapter_name": lambda doc: f"{doc.get('kandam','')}",
+        "title": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
+        "author": lambda doc: "Kamban",
+        "unit_index": "verse_number",
+        "verse": lambda doc: int(doc.get("verse_number", "0")),
+        "relative_path": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
+    },
+    "metadata_fields": [
+        {
+            "name": "kandam",
+            "label": "Kandam",
+            "datatype": "str",
+            "description": "The name of the Kandam or the chapter.",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_list_of_kandams(),
+        },
+        {
+            "name": "padalam_en",
+            "label": "Chapter Name",
+            "datatype": "str",
+            "description": "The name of the Padalam (Episode) in English.",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_list_of_padalams(),
+        },
+        {
+            "name": "padalam_ta",
+            "label": "Padalam name in Tamil",
+            "datatype": "str",
+            "description": "The name of the Padalam (Episode) in Tamil.",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_list_of_padalams_in_tamil(),
+        },
+        {
+            "name": "page",
+            "datatype": "int",
+            "label": "Page Number",
+            "description": "Page number from the source",
+            "show_as_filter": True,
+        },
+        {
+            "name": "verse_number",
+            "datatype": "int",
+            "label": "Verse Number",
+            "description": "Verse Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/kamba_ramayanam.pdf",
+    "source": "https://www.hindupedia.com/images/1/13/Kamba_Ramayanam_I.pdf",
+    "language": "tamil",
+    "example_labels": [
+        "About Jatayu",
+        "About Hanuman",
+        "About Vali",
+        "About Sita",
+        "About Ravana",
+    ],
+    "examples": [
+        "What is the significance of Jatayu? show some sanskrit verses to support the argument",
+        "Show some verses where Hanuman is mentioned",
+        "How did Rama kill Vali",
+        "How was Sita abducted",
+        "How did Rama kill Ravana?",
+    ],
+    "llm_hints": [],
+}

modules/config/katakam.py

@@ -0,0 +1,81 @@
+katakam_config = {
+    "name": "katakam",
+    "title": "Katakam",
+    "output_dir": "./output/katakam",
+    "collection_name": "katakam",
+    "collection_embedding_fn": "openai",
+    "unit": "panchadhi",
+    "unit_field": "panchadhi",
+    # "chapter_order": lambda: get_chapter_order_from_katakam(),
+    "field_mapping": {
+        "text": "sanskrit",
+        "unit_index": "panchadhi",
+        "transliteration": "transliteration",
+        "chapter_name": lambda doc: (
+            doc.get("prapaatakam_name")
+            if doc.get("prapaatakam_name") and doc.get("prapaatakam_name") != "-"
+            else "॥ काठकम् ॥"
+        ),
+        "relative_path": lambda doc: (
+            "-"
+            if not doc.get("panchadhi") or doc.get("panchadhi") == "-"
+            else f"{doc.get('prapaatakam')}.{doc.get('anuvakam')}.{doc.get('panchadhi_index')} || {doc.get('panchadhi_sa')} ||"
+        ),
+    },
+    "metadata_fields": [
+        {
+            "name": "prapaatakam",
+            "datatype": "int",
+            "label": "Prapaatakam Number",
+            "description": "Prapaatakam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "anuvakam",
+            "datatype": "int",
+            "label": "Anuvakam Number",
+            "description": "Anuvakam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "panchadhi_index",
+            "datatype": "int",
+            "label": "Relative Panchadhi Number",
+            "description": "Relative Panchadhi Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "panchadhi",
+            "datatype": "int",
+            "label": "Absolute Panchadhi Number",
+            "description": "Absolute Panchadhi Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "transliteration",
+            "label": "Transliteration in english",
+            "datatype": "str",
+            "description": "The original sloka transliterated in English.",
+        },
+    ],
+    "pdf_path": "./data/katakam.pdf",
+    "source": "https://sanskritdocuments.org/doc_veda/taittirIyabrAhmaNam.html",
+    "language": "san+eng",
+    "example_labels": [
+        "Katakam",
+    ],
+    "examples": [
+        "Show some verses from Katakam",
+    ],
+    "llm_hints": [],
+}

modules/config/pancha_sooktham.py

@@ -0,0 +1,57 @@
+from modules.scripture_helpers.pancha_sooktham_helper import (
+    get_chapter_order_from_pancha_sooktham,
+    get_pancha_sooktham_chapters,
+)
+pancha_sooktham_config = {
+    "name": "pancha_sooktham",
+    "title": "Pancha Sooktham",
+    "output_dir": "./output/pancha_sooktham",
+    "collection_name": "pancha_sooktham",
+    "collection_embedding_fn": "openai",
+    "unit": "verse",
+    "unit_field": "relative_verse_number",
+    "chapter_order": lambda: get_chapter_order_from_pancha_sooktham(),
+    "field_mapping": {
+        "text": "sanskrit",
+        "unit_index": "relative_verse_number",
+        "transliteration": lambda doc: f"English: \n{doc.get('english')}\n\nதமிழ்: \n{doc.get('tamil')}\n\nతెలుగు: \n{doc.get('telugu')}\n\nಕನ್ನಡ: \n{doc.get('kannada')}",
+        "chapter_name": "chapter_title",
+        "relative_path": lambda doc: f"{doc.get('chapter_title',"")}-{doc.get('relative_verse_number',"")}",
+    },
+    "metadata_fields": [
+        {
+            "name": "chapter_title",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Name of the Chapter",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_pancha_sooktham_chapters(),
+            "is_unique": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "relative_verse_number",
+            "label": "Relative Verse Number",
+            "datatype": "int",
+            "description": "The relative verse number of the sloka inside the chapter.",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/pancha_sooktham.pdf",
+    "source": "https://stotranidhi.com/",
+    "language": "san+eng",
+    "example_labels": [
+        "Purusha Sooktham",
+    ],
+    "examples": [
+        "Show some verses from purusha sooktham",
+    ],
+    "llm_hints": [],
+}

modules/config/shanthi_panchakam.py

@@ -0,0 +1,58 @@
+from modules.scripture_helpers.shanthi_panchakam_helper import (
+    get_chapter_order_from_shanthi_panchakam,
+    get_shanthi_panchakam_chapters,
+)
+
+shanthi_panchakam_config = {
+    "name": "shanthi_panchakam",
+    "title": "Shanthi Panchakam",
+    "output_dir": "./output/shanthi_panchakam",
+    "collection_name": "shanthi_panchakam",
+    "collection_embedding_fn": "openai",
+    "unit": "verse",
+    "unit_field": "relative_verse_number",
+    "chapter_order": lambda: get_chapter_order_from_shanthi_panchakam(),
+    "field_mapping": {
+        "text": "sanskrit",
+        "unit_index": "relative_verse_number",
+        "transliteration": lambda doc: f"English: \n{doc.get('english')}\n\nதமிழ்: \n{doc.get('tamil')}\n\nతెలుగు: \n{doc.get('telugu')}\n\nಕನ್ನಡ: \n{doc.get('kannada')}",
+        "chapter_name": "chapter_title",
+        "relative_path": lambda doc: f"{doc.get('chapter_title',"")}-{doc.get('relative_verse_number',"")}",
+    },
+    "metadata_fields": [
+        {
+            "name": "chapter_title",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Name of the Chapter",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_shanthi_panchakam_chapters(),
+            "is_unique": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "relative_verse_number",
+            "label": "Relative Verse Number",
+            "datatype": "int",
+            "description": "The relative verse number of the sloka inside the chapter.",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/shanthi_panchakam.pdf",
+    "source": "https://vignanam.org/english/shanti-panchakam.html",
+    "language": "san+eng",
+    "example_labels": [
+        "Shanthi Panchakam",
+    ],
+    "examples": [
+        "Show some verses from Shanthi Panchakam",
+    ],
+    "llm_hints": [],
+}

modules/config/shukla_yajur_vedam.py

@@ -0,0 +1,37 @@
+shukla_yajur_vedam_config = {
+    "name": "shukla_yajur_vedam",
+    "title": "Shukla Yajur Vedam",
+    "output_dir": "./output/shukla_yajur_vedam",
+    "collection_name": "shukla_yajur_vedam",
+    "unit": "page",
+    "metadata_fields": [
+        {
+            "name": "file",
+            "label": "File Name",
+            "datatype": "str",
+            "description": "name of the file from which the information was extracted",
+        },
+        {
+            "name": "page",
+            "datatype": "int",
+            "label": "Page Number",
+            "description": "Page number from the source",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/shukla-yajur-veda.pdf",
+    "source": "https://www.thearyasamaj.org/uploads/book/2014/04/R1sSjG_eLb_sub_406_yajurveda.pdf",
+    "language": "san+eng",
+    "example_labels": [
+        "About Vedam",
+        "About the five elements",
+        "About Brahma",
+    ],
+    "examples": [
+        "Gist of Shukla Yajur Vedam. Give me some sanskrit verses.",
+        "What is the significance of fire and water. show some sanskrit verses",
+        "Brahma",
+    ],
+    "llm_hints": [],
+}

modules/config/sri_stavam.py

@@ -0,0 +1,59 @@
+sri_stavam_config = {
+    "name": "sri_stavam",
+    "title": "Sri Stavam by Sri Koorathazhwan",
+    "output_dir": "./output/sri_stavam",
+    "collection_name": "sri_stavam",
+    "unit": "slokam",
+    "unit_field": "sloka_number",
+    "field_mapping": {
+        "text": "sanskrit",
+        "title": lambda doc: f"Verse {doc.get('verse','')}",
+        "unit_index": "sloka_number",
+        "transliteration": "transliteration",
+        "word_by_word_native": "meaning_short",
+        "author": lambda doc: f"Sri Koorathazhwan",
+        "relative_path": lambda doc: f"Sri Stavam-{doc.get("sloka_number","")}",
+    },
+    "metadata_fields": [
+        {
+            "name": "sloka_number",
+            "label": "Slokam Number",
+            "datatype": "int",
+            "description": "The index of the sloka or verse",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "meaning_short",
+            "label": "Short meaning",
+            "datatype": "str",
+            "description": "A short meaning of the sanskrit verse in English.",
+            "show_as_filter": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "Verse in sanskrit",
+        },
+        {
+            "name": "transliteration",
+            "label": "English Transliteration",
+            "datatype": "str",
+            "description": "Verse transliterated to English",
+            "show_as_filter": True,
+        },
+    ],
+    "pdf_path": "./data/sri_stavam.pdf",
+    "source": "https://www.sadagopan.org/ebook/pdf/Sri%20Stavam.pdf",
+    "language": "san+eng",
+    "example_labels": ["Recite a sloka", "Commentary", "Role of Sridevi"],
+    "examples": [
+        "Recite the 1st sloka from Sri Stavam",
+        "Show detailed commentary for sloka 2 from Sri Stavam",
+        "What is the role of Sri Devi in the universe according to the Sri Stavam?",
+    ],
+    "llm_hints": [
+        "if the user asks for nth sloka, do a metadata search on the `verse` field."
+    ],
+}

modules/config/taitriya_brahmanam.py

@@ -0,0 +1,93 @@
+taitriya_brahmanam_config = {
+    "name": "taitriya_brahmanam",
+    "title": "Taitriya Brahmanam",
+    "output_dir": "./output/taitriya_brahmanam",
+    "collection_name": "taitriya_brahmanam",
+    "collection_embedding_fn": "openai",
+    "unit": "panchadhi",
+    "unit_field": "panchadhi",
+    # "chapter_order": lambda: get_chapter_order_from_taitriya_brahmanam(),
+    "field_mapping": {
+        "text": "sanskrit",
+        "unit_index": "panchadhi",
+        "transliteration": "transliteration",
+        "chapter_name": lambda doc: (
+            doc.get("prapaatakam_name")
+            if doc.get("prapaatakam_name") and doc.get("prapaatakam_name") != "-"
+            else (
+                doc.get("ashtakam_name")
+                if doc.get("ashtakam_name") and doc.get("ashtakam_name") != "-"
+                else "-"
+            )
+        ),
+        "relative_path": lambda doc: (
+            "-"
+            if not doc.get("panchadhi") or doc.get("panchadhi") == "-"
+            else f"{doc.get('ashtakam')}.{doc.get('prapaatakam')}.{doc.get('anuvakam')}.{doc.get('panchadhi_index')} || {doc.get('panchadhi_sa')} ||"
+        ),
+    },
+    "metadata_fields": [
+        {
+            "name": "ashtakam",
+            "datatype": "int",
+            "label": "Ashtakam Number",
+            "description": "Ashtakam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "prapaatakam",
+            "datatype": "int",
+            "label": "Prapaatakam Number",
+            "description": "Prapaatakam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "anuvakam",
+            "datatype": "int",
+            "label": "Anuvakam Number",
+            "description": "Anuvakam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "panchadhi_index",
+            "datatype": "int",
+            "label": "Relative Panchadhi Number",
+            "description": "Relative Panchadhi Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "panchadhi",
+            "datatype": "int",
+            "label": "Absolute Panchadhi Number",
+            "description": "Absolute Panchadhi Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "transliteration",
+            "label": "Transliteration in english",
+            "datatype": "str",
+            "description": "The original sloka transliterated in English.",
+        },
+    ],
+    "pdf_path": "./data/taitriya_brahmanam.pdf",
+    "source": "https://sanskritdocuments.org/doc_veda/taittirIyabrAhmaNam.html",
+    "language": "san+eng",
+    "example_labels": [
+        "Taitriya Brahmanam",
+    ],
+    "examples": [
+        "Show some verses from Taitriya Brahmanam",
+    ],
+    "llm_hints": [],
+}

modules/config/taitriya_samhitha.py

@@ -0,0 +1,93 @@
+taitriya_samhitha_config = {
+    "name": "taitriya_samhitha",
+    "title": "Taitriya Samhitha",
+    "output_dir": "./output/taitriya_samhitha",
+    "collection_name": "taitriya_samhitha",
+    "collection_embedding_fn": "openai",
+    "unit": "panchadhi",
+    "unit_field": "panchadhi",
+    # "chapter_order": lambda: get_chapter_order_from_taitriya_samhitha(),
+    "field_mapping": {
+        "text": "sanskrit",
+        "unit_index": "panchadhi",
+        "transliteration": "transliteration",
+        "chapter_name": lambda doc: (
+            doc.get("prashnam_name")
+            if doc.get("prashnam_name") and doc.get("prashnam_name") != "-"
+            else (
+                doc.get("kandam_name")
+                if doc.get("kandam_name") and doc.get("kandam_name") != "-"
+                else "-"
+            )
+        ),
+        "relative_path": lambda doc: (
+            "-"
+            if not doc.get("panchadhi") or doc.get("panchadhi") == "-"
+            else f"{doc.get('kandam')}.{doc.get('prashnam')}.{doc.get('anuvakam')}.{doc.get('panchadhi_index')} || {doc.get('panchadhi_sa')} ||"
+        ),
+    },
+    "metadata_fields": [
+        {
+            "name": "kandam",
+            "datatype": "int",
+            "label": "Kandam Number",
+            "description": "Kandam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "prashnam",
+            "datatype": "int",
+            "label": "Prashnam Number",
+            "description": "Prashnam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "anuvakam",
+            "datatype": "int",
+            "label": "Anuvakam Number",
+            "description": "Anuvakam Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "panchadhi_index",
+            "datatype": "int",
+            "label": "Relative Panchadhi Number",
+            "description": "Relative Panchadhi Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "panchadhi",
+            "datatype": "int",
+            "label": "Absolute Panchadhi Number",
+            "description": "Absolute Panchadhi Number",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "transliteration",
+            "label": "Transliteration in english",
+            "datatype": "str",
+            "description": "The original sloka transliterated in English.",
+        },
+    ],
+    "pdf_path": "./data/taitriya_samhitha.pdf",
+    "source": "https://sanskritdocuments.org/doc_veda/taittirIyasamhitA.html",
+    "language": "san+eng",
+    "example_labels": [
+        "Taitriya Samhitha",
+    ],
+    "examples": [
+        "Show some verses from Taitriya Samhitha",
+    ],
+    "llm_hints": [],
+}

modules/config/taitriya_upanishad.py

@@ -0,0 +1,58 @@
+from modules.scripture_helpers.taitriya_upanishad_helper import (
+    get_chapter_order_from_taitriya_upanishad,
+    get_taitriya_upanishad_chapters,
+)
+
+taitriya_upanishad_config = {
+            "name": "taitriya_upanishad",
+            "title": "Taitriya Upanishad",
+            "output_dir": "./output/taitriya_upanishad",
+            "collection_name": "taitriya_upanishad",
+            "collection_embedding_fn": "openai",
+            "unit": "verse",
+            "unit_field": "relative_verse_number",
+            "chapter_order": lambda: get_chapter_order_from_taitriya_upanishad(),
+            "field_mapping": {
+                "text": "sanskrit",
+                "unit_index": "relative_verse_number",
+                "transliteration": lambda doc: f"English: \n{doc.get('english')}\n\nதமிழ்: \n{doc.get('tamil')}\n\nతెలుగు: \n{doc.get('telugu')}\n\nಕನ್ನಡ: \n{doc.get('kannada')}",
+                "chapter_name": "chapter_title",
+                "relative_path": lambda doc: f"{doc.get('chapter_title',"")}-{doc.get('relative_verse_number',"")}",
+            },
+            "metadata_fields": [
+                {
+                    "name": "chapter_title",
+                    "datatype": "str",
+                    "label": "Chapter Name",
+                    "description": "Name of the Chapter",
+                    "show_as_filter": True,
+                    "component": "dropdown",
+                    "lov": lambda: get_taitriya_upanishad_chapters(),
+                    "is_unique": True,
+                },
+                {
+                    "name": "sanskrit",
+                    "label": "Lyrics in sanskrit",
+                    "datatype": "str",
+                    "description": "The original sloka in sanskrit.",
+                },
+                {
+                    "name": "relative_verse_number",
+                    "label": "Relative Verse Number",
+                    "datatype": "int",
+                    "description": "The relative verse number of the sloka inside the chapter.",
+                    "show_as_filter": True,
+                    "is_unique": True,
+                },
+            ],
+            "pdf_path": "./data/taitriya_upanishad.pdf",
+            "source": "https://stotranidhi.com/",
+            "language": "san+eng",
+            "example_labels": [
+                "Seeksha Valli",
+            ],
+            "examples": [
+                "Show some verses from Seeksha Valli",
+            ],
+            "llm_hints": [],
+        }

modules/config/valmiki_ramayanam.py

@@ -0,0 +1,46 @@
+valmiki_ramayanam_config = {
+    "name": "valmiki_ramayanam",
+    "title": "Valmiki Ramayanam",
+    "output_dir": "./output/valmiki_ramayanam",
+    "collection_name": "valmiki_ramayanam_openai",
+    "collection_embedding_fn": "openai",
+    "unit": "page",
+    "metadata_fields": [
+        {
+            "name": "file",
+            "label": "File Name",
+            "datatype": "str",
+            "description": "name of the file from which the information was extracted",
+        },
+        {
+            "name": "page",
+            "datatype": "int",
+            "label": "Page Number",
+            "description": "Page number from the source",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/valmiki_ramayanam.pdf",
+    "source": "https://ia800509.us.archive.org/28/items/valmiki-ramayana-gita-press-english/Valmiki%20Ramayana%20Gita%20Press%20English.pdf",
+    "language": "san+eng",
+    "example_labels": [
+        "About Jatayu",
+        "About Hanuman",
+        "About Vali",
+        "About Sita",
+        "About Ravana",
+        "A slokam by name",
+        "Vibheeshana sharanagathi slokam",
+    ],
+    "examples": [
+        "What is the significance of Jatayu? show some sanskrit verses to support the argument",
+        "Show some verses where Hanuman is mentioned",
+        "How did Rama kill Vali",
+        "How was Sita abducted",
+        "How did Rama kill Ravana?",
+        "explain sakrudeva prapannaaya shlokam in ramayana",
+        "give the shlokam in ramayanam that vibheeshana uses to perform sharanagathi to rama, give the sanskrit shlokam and its meaning",
+    ],
+    "llm_hints": [],
+}

modules/config/vishnu_puranam.py

@@ -0,0 +1,42 @@
+vishnu_puranam_config = {
+    "name": "vishnu_puranam",
+    "title": "Sri Vishnu Puranam",
+    "output_dir": "./output/vishnu_puranam",
+    "collection_name": "vishnu_puranam_openai",
+    "collection_embedding_fn": "openai",
+    "unit": "page",
+    "metadata_fields": [
+        {
+            "name": "file",
+            "label": "File Name",
+            "datatype": "str",
+            "description": "name of the file from which the information was extracted",
+        },
+        {
+            "name": "page",
+            "datatype": "int",
+            "label": "Page Number",
+            "description": "Page number from the source",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/vishnu_puranam.pdf",
+    "source": "https://dn720005.ca.archive.org/0/items/vishnu-purana-sanskrit-english-ocr/VISHNU-PURANA-Sanskrit-English-OCR.pdf",
+    "language": "san+eng",
+    "example_labels": [
+        "Vishnu's form",
+        "About the five elements",
+        "About Garuda",
+        "Weapons of Vishnu",
+        "Vishnu's form (all scriptures)",
+    ],
+    "examples": [
+        "describe Vishnu's form as defined in vishnu puranam",
+        "five elements and their significance as per vishnu puranam",
+        "What is the significance of Garuda? Show some verses from vishnu puranam that describe him.",
+        "What weapons does Vishnu hold as mentioned in vishnu puranam?",
+        "How is the form of Vishnu described across the scriptures?",
+    ],
+    "llm_hints": [],
+}

modules/config/vishnu_sahasranamam.py

@@ -0,0 +1,78 @@
+from modules.scripture_helpers.vishnu_sahasranamam_helper import (
+    get_chapter_order_from_sahasranamam,
+    get_chapters_from_sahasranamam,
+)
+
+vishnu_sahasranamam_config = {
+    "name": "vishnu_sahasranamam",
+    "title": "Vishnu Sahasranamam",
+    "output_dir": "./output/vishnu_sahasranamam",
+    "collection_name": "vishnu_sahasranamam_openai",
+    "collection_embedding_fn": "openai",
+    "unit": "verse",
+    "chapter_order": lambda: get_chapter_order_from_sahasranamam(),
+    "field_mapping": {
+        "text": "sanskrit",
+        "title": lambda doc: f"Verse {doc.get('verse','')}",
+        "unit_index": "verse",
+        "transliteration": "transliteration",
+        "translation": "translation",
+        "author": lambda doc: f"Sage Vyasa",
+        "chapter_name": "chapter",
+        "relative_path": lambda doc: f"{doc.get("chapter","")}-{doc.get("verse","")}",
+    },
+    "metadata_fields": [
+        {
+            "name": "chapter",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Name of the Chapter",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: get_chapters_from_sahasranamam(),
+            "is_unique": True,
+        },
+        {
+            "name": "page_number",
+            "datatype": "int",
+            "label": "Page Number",
+            "description": "Page number from the source",
+        },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+        },
+        {
+            "name": "translation",
+            "label": "English Translation",
+            "datatype": "str",
+            "description": "The english translation.",
+        },
+        {
+            "name": "transliteration",
+            "label": "English Transliteration",
+            "datatype": "str",
+            "description": "The english transliteration.",
+        },
+        {
+            "name": "verse",
+            "label": "Verse Number",
+            "datatype": "int",
+            "description": "The verse number of the sloka.",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+    ],
+    "pdf_path": "./data/vishnu_sahasranamam.pdf",
+    "source": "https://www.swami-krishnananda.org/vishnu/Sri_Vishnu_Sahasranama_Stotram.pdf",
+    "language": "san+eng",
+    "example_labels": ["Vanamali", "1000 names", "Sanskrit text search"],
+    "examples": [
+        "Vanamali",
+        "Show some of the 1000 names of Vishnu along with their meaning",
+        "show the verse that begins with शुक्लाम्बरधरं",
+    ],
+    "llm_hints": [],
+}

modules/config/yt_metadata.py

@@ -0,0 +1,62 @@
+yt_metadata_config = {
+    "name": "yt_metadata",
+    "title": "Sampradayam in YouTube",
+    "output_dir": "./output/yt_metadata",
+    "collection_name": "yt_metadata",
+    "collection_embedding_fn": "openai",
+    "unit": "video",
+    "field_mapping": {
+        "text": "description",
+        "title": "video_title",
+        "author": "channel_title",
+        "reference_link": lambda doc: f"https://www.youtube.com/watch?v={doc.get('video_id','')}",
+    },
+    "metadata_fields": [
+        {
+            "name": "video_id",
+            "label": "Video ID",
+            "datatype": "str",
+            "description": "The video id as in YouTube",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "video_title",
+            "label": "Video Title",
+            "datatype": "str",
+            "description": "The title of the video as in YouTube",
+            "show_as_filter": True,
+        },
+        {
+            "name": "description",
+            "label": "Video Description",
+            "datatype": "str",
+            "description": "Description as in YouTube",
+            "show_as_filter": True,
+        },
+        {
+            "name": "channel_url",
+            "label": "Channel URL",
+            "datatype": "str",
+            "description": "URL of the YouTube Channel",
+        },
+        {
+            "name": "channel_title",
+            "label": "Channel Title",
+            "datatype": "str",
+            "description": "Title of the YouTube Channel",
+            "show_as_filter": True,
+        },
+    ],
+    "pdf_path": "./data/none.pdf",
+    "source": "https://youtube.com",
+    "language": "san+eng+tam",
+    "example_labels": ["Srirangam", "Pasuram video"],
+    "examples": [
+        "Show me YouTube videos that talk about Srirangam",
+        "Show me lyrics of 1st pasuram of 1st decade in the 4st Thiruvaimozhi. Also show the related youtube videos.",
+    ],
+    "llm_hints": [
+        "if the user asks for YouTube videos, DO NOT do a web search, instead do a search on this collection."
+    ],
+}