Upload folder using huggingface_hub

config.py

@@ -3,15 +3,7 @@ from typing import List, Dict
 
 
 class SanatanConfig:
-    # shuklaYajurVedamPdfPath: str = "./data/shukla-yajur-veda.pdf"
-    # shuklaYajurVedamSmallPdfPath: str = "./data/shukla-yajur-veda-small.pdf"
-    # vishnuPuranamPdfPath = "./data/vishnu_puranam.pdf"
-    # datastores = [{"name": "sanskrit_001", "dbStorePath": "./chromadb-store"}, {"name": "nalayiram", "dbStorePath": "./chromadb-store-4000"}]
     dbStorePath: str = "./chromadb-store"
-    # shuklaYajurVedamCollectionName: str = "shukla_yajur_vedam"
-    # vishnuPuranamCollectionName: str = "vishnu_puranam"
-    # shuklaYajurVedamOutputDir = "./output/shukla_yajur_vedam"
-    # vishnuPuranamOutputDir = "./output/vishnu_puranam"
     scriptures = [
         {
             "name": "vishnu_puranam",
@@ -203,6 +195,29 @@ class SanatanConfig:
             "collection_name": "divya_prabandham",
             "collection_embedding_fn": "openai",
             "unit": "verse",
+            "field_mapping": {
+                "text": "pasuram_ta",
+                "title": lambda doc: f"{doc.get('prabandham_name','')} {doc.get('chapter','')}-{doc.get('decade','')}:{doc.get('position_in_chapter','')}",
+                "word_by_word_native": "wbw_ta",
+                "unit_index": "verse",
+                "transliteration": "pasuram_en",
+                "reference_link": "html_url",
+                "author": "azhwar_name",
+                "chapter_name": "prabandham_name",
+                "relative_path": lambda doc: "-".join(
+                    filter(
+                        None,
+                        [
+                            doc.get("prabandham_name", ""),
+                            *(
+                                str(doc.get(k))
+                                for k in ["decade", "chapter", "position_in_chapter"]
+                                if doc.get(k, -1) != -1
+                            ),
+                        ],
+                    )
+                ),
+            },
             "metadata_fields": [
                 {
                     "name": "prabandham_code",
@@ -246,7 +261,7 @@ class SanatanConfig:
                 {
                     "name": "verse",
                     "datatype": "int",
-                    "is_unique" : True,
+                    "is_unique": True,
                     "description": (
                         "Absolute verse number or pasuram number. Each verse has a unique number."
                         # "Use it only when a specific prabandham name is NOT mentioned in the user query."
@@ -574,3 +589,60 @@ class SanatanConfig:
         for s in self.scriptures:
             filtered.append({k: s[k] for k in fields_to_keep if k in s})
         return filtered
+
+    def canonicalize_document(
+        self, scripture_name: str, document_text: str, metadata_doc: dict
+    ):
+        """
+        Convert scripture-specific document to a flattened canonical form.
+        Supports static strings or lambdas in field mapping.
+        Only allows keys from the allowed canonical fields list.
+        """
+        allowed_keys = {
+            "verse",
+            "text",
+            "title",
+            "unit",
+            "unit_index",
+            "word_by_word_native",
+            "transliteration",
+            "reference_link",
+            "author",
+            "chapter_name",
+            "relative_path",
+        }
+
+        config = next((s for s in self.scriptures if s["name"] == scripture_name), None)
+        if not config:
+            raise ValueError(f"Unknown scripture: {scripture_name}")
+
+        mapping = config.get("field_mapping", {})
+
+        def resolve_field(field):
+            """Resolve a field: string key or lambda"""
+            if callable(field):
+                try:
+                    return field(metadata_doc)
+                except Exception:
+                    return None
+            elif isinstance(field, str):
+                return metadata_doc.get(field)
+            return None
+
+        canonical_doc = {}
+        for key, field in mapping.items():
+            if key in allowed_keys:  # only include allowed canonical keys
+                canonical_doc[key] = resolve_field(field)
+
+        # optionally add global fields from config
+        canonical_doc["scripture_name"] = config.get("name")
+        canonical_doc["scripture_title"] = config.get("title")
+        canonical_doc["source"] = config.get("source")
+        canonical_doc["language"] = config.get("language")
+        canonical_doc["unit"] = config.get("unit")
+        canonical_doc["document"] = document_text
+        if canonical_doc["text"] == "-" or canonical_doc["text"] is None:
+            canonical_doc["text"] = canonical_doc["document"]
+        canonical_doc["verse"] = resolve_field("verse")
+
+        return canonical_doc

db.py

@@ -112,6 +112,50 @@ class SanatanDatabase:
                 n_results=n_results,
             )
 
+    def fetch_document_by_index(self, collection_name: str, index: int, unit_name : str):
+        """
+        Fetch one document at a time from a ChromaDB collection using pagination (index = 0-based).
+
+        Args:
+            collection_name: Name of the ChromaDB collection.
+            index: Zero-based index of the document to fetch.
+
+        Returns:
+            dict: {
+                "document": <document_text>,
+                <metadata_key_1>: <value>,
+                <metadata_key_2>: <value>,
+                ...
+            }
+            Or a dict with "error" key if something went wrong.
+        """
+        logger.info("Fetch document #%d from [%s]", index, collection_name)
+        collection = self.chroma_client.get_or_create_collection(name=collection_name)
+
+        try:
+            response = collection.get(
+                limit=1,
+                # offset=index,  # pagination via offset
+                include=["metadatas", "documents"],
+                where={unit_name: index}
+            )
+        except Exception as e:
+            logger.error("Error fetching document: %s", e)
+            return {"error": f"There was an error fetching the document: {str(e)}"}
+
+        documents = response.get("documents", [])
+        metadatas = response.get("metadatas", [])
+
+        if documents:
+            # merge document text with metadata
+            result = {"document": documents[0]}
+            if metadatas:
+                result.update(metadatas[0])
+            return result
+        else:
+            return {"error": "No data available."}
+
+
     def search_semantic(
         self,
         collection_name: str,

main.py

@@ -31,4 +31,4 @@ async def log_requests(request: Request, call_next):
     return response
 
 if __name__ == "__main__":
-    uvicorn.run("main:app", host="0.0.0.0", port=7860)
+    uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=True)

server.py

@@ -180,10 +180,75 @@ async def handle_quiz_eval(payload: QuizEvalPayload, request: Request):
     print(result.model_dump_json(indent=1))
     return result
 
+
 @router.get("/scriptures")
 async def handle_get_scriptures():
     return_values = {}
     for scripture in SanatanConfig().scriptures:
-        if scripture['collection_name'] != "yt_metadata":
-            return_values[scripture['collection_name']] = scripture['title']
+        if scripture["collection_name"] != "yt_metadata":
+            return_values[scripture["collection_name"]] = scripture["title"]
     return return_values
+
+
+class ScriptureRequest(BaseModel):
+    scripture_name: str
+    unit_index: int
+
+
+@router.post("/scripture")
+async def get_scripture(req: ScriptureRequest):
+    """
+    Return a scripture unit (page or verse, based on config),
+    including all metadata fields separately.
+    """
+    print("received request to fetch scripture.", req)
+
+    # find config entry for the scripture
+    config = next(
+        (s for s in SanatanConfig().scriptures if s["name"] == req.scripture_name), None
+    )
+    if not config:
+        return {"error": f"Scripture '{req.scripture_name}' not found"}
+
+    # fetch the raw document from DB
+    raw_doc = SanatanDatabase().fetch_document_by_index(
+        collection_name=config["collection_name"],
+        index=req.unit_index,
+        unit_name=config["unit"]
+    )
+
+    if not raw_doc or isinstance(raw_doc, str):
+        return {"error": f"No data available for unit {req.unit_index}"}
+
+    # canonicalize it
+    canonical_doc = SanatanConfig().canonicalize_document(
+        scripture_name=req.scripture_name,
+        document_text=raw_doc.get("document", ""),
+        metadata_doc=raw_doc,
+    )
+
+    # add unit index & total units (so Flutter can paginate)
+    canonical_doc["unit_index"] = req.unit_index
+    canonical_doc["total"] = SanatanDatabase().count(config["collection_name"])
+
+    print("canonical_doc = ", canonical_doc)
+    return canonical_doc
+
+
+@router.get("/scripture_configs")
+async def get_scripture_configs():
+    scriptures = []
+    for s in SanatanConfig().scriptures:
+        num_units = SanatanDatabase().count(
+            collection_name=s["collection_name"]
+        )
+
+        scriptures.append(
+            {
+                "name": s["name"],  # e.g. "bhagavad_gita"
+                "title": s["title"],  # e.g. "Bhagavad Gita"
+                "unit": s["unit"],  # e.g. "verse" or "page"
+                "total" : num_units
+            }
+        )
+    return {"scriptures": scriptures}