v0.1

.gitignore

@@ -0,0 +1,16 @@
+src/.env
+pytest.ini
+logger.log
+.pytest_cache/**
+**/__pycache__/**
+.vscode/**
+**/.env
+src/baseInfra/*.env
+**/*.env*
+**/*.pyc
+src/baseInfra/**.env**
+src/baseInfra/__pycache__/*.pyc
+src/*.old.py
+src/toolSelector.py
+test/__pycache__/*.pyc
+src/baseInfra/memoryManager.py

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.11.0
+
+ENV PYTHONUNBUFFERED 1
+
+EXPOSE 8000
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+ENV TZ="Asia/Kolkata"
+WORKDIR $HOME/app
+
+COPY requirements.txt ./
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt
+
+
+COPY --chown=user . $HOME/app
+ENV PYTHONPATH=.:/home/user/app/src:$PYTHONPATH
+RUN ls -al
+RUN python --version
+#CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"]
+
+CMD ["python", "./src/main.py", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -5,6 +5,7 @@ colorFrom: pink
 colorTo: purple
 sdk: docker
 pinned: false
+app_port: 8000
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

logging_config.yaml

@@ -0,0 +1,37 @@
+version: 1
+formatters:
+  simple:
+    format: '%(asctime)s - %(filename)s - %(funcName)10s() - %(lineno)d - %(levelname)s - %(message)s'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: simple
+    stream: ext://sys.stdout
+  logfile:
+    class: logging.FileHandler
+    level: DEBUG
+    formatter: simple
+    filename: logger.log
+    encoding: utf8
+loggers:    #These are for individual files and then finally for root
+  objectiveHandler:
+    level: DEBUG
+    handlers: [console,logfile]
+    propagate: no
+  taskExecutor:
+    level: DEBUG
+    handlers: [console,logfile]
+    propagate: no
+  taskPlanner:
+    level: DEBUG
+    handlers: [console,logfile]
+    propagate: no
+  connectionpool:
+    level: ERROR
+    handlers: [console,logfile]
+    propagate: no
+  root:
+    level: DEBUG
+    handlers: [console,logfile]
+  

main.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+import logging
+import fastapi
+from fastapi import Body, Depends
+import uvicorn
+from fastapi import HTTPException , status
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI as Response
+from sse_starlette.sse import EventSourceResponse
+from starlette.responses import StreamingResponse
+from starlette.requests import Request
+from pydantic import BaseModel
+from enum import Enum
+from typing import List, Dict, Any, Generator, Optional, cast, Callable
+from indexer import *
+
+
+
+async def catch_exceptions_middleware(
+    request: Request, call_next: Callable[[Request], Any]
+) -> Response:
+    try:
+        return await call_next(request)
+    except Exception as e:
+        return JSONResponse(content={"error": repr(e)}, status_code=500)
+
+
+
+app = fastapi.FastAPI(title="Maya Persistet")
+app.middleware("http")(catch_exceptions_middleware)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+api_base="/api/v1"
+
+@app.post(api_base+"/getMatchingDocs")
+async def get_matching_docs(inStr: str ) -> Any:
+    """
+    """
+    return getRelevantDocs(inStr)
+
+
+
+
+print(__name__)
+
+if __name__ == '__main__' or __name__ == "src.main":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

requirements.txt

@@ -0,0 +1,20 @@
+uvicorn
+bs4
+lxml
+fastapi
+loguru
+chromadb
+langchain
+sentence_transformers
+InstructorEmbedding
+sse_starlette
+dropbox
+firebase
+together
+python-dotenv
+pytest
+pytest-env
+pydantic==1.10.9
+openapi-schema-pydantic
+markdown
+google-generativeai

src/baseInfra/dbInterface.py

@@ -0,0 +1,97 @@
+from firebase import firebase
+import os,json
+import baseInfra.firebase_handler as fbh
+
+secret=os.environ['FIREBASE_TOKEN'] 
+user_id="131251"
+auth=firebase.FirebaseAuthentication(secret=secret,email="anubhav77@gmail.com",extra={"id":user_id})
+fb_url="https://device-1a455.firebaseio.com"
+fb= firebase.FirebaseApplication(fb_url,auth)
+base_dir="/users/"+user_id
+
+class DbInterface:
+    """
+    A database interface.
+
+    This class provides a dummy implementation of a database interface. It has two methods: `get_task_list()` and `update_task_list()`.
+    """
+    def __init__(self):
+        self.base_dir=base_dir
+        self.conf_loc="config"
+        self.cache_loc="vs_cache"
+
+    def _set_base_dir(self,sub_dir):
+        """
+        This internal method is primarily for unit testcases
+        Adds extra subdir to the path
+        """
+        if sub_dir.startswith("/"):
+            self.base_dir=base_dir+sub_dir
+        else:
+            self.base_dir=base_dir+"/"+sub_dir
+        if self.base_dir.endswith("/"):
+            self.base_dir=self.base_dir[:-1]
+        return self.base_dir
+
+
+    def get_config(self,path):
+        """
+        Gets the config from the database given path
+        This is used for getting configs of executor, planner or tools
+
+        Returns:
+            The json config object.
+
+        """
+        print(self.base_dir+"/"+self.conf_loc)
+        print(path)
+        config = fb.get(self.base_dir+"/"+self.conf_loc, path)
+        if config is None:
+            return {}
+        return config
+    
+    def get_matching_cache(self,input):
+        """
+        Gets the matching cache from the database given input
+        This is used for getting similar documents from the vector store
+
+        Returns:
+            The json cache object.
+
+        """
+        fb_key=fbh.convert_to_firebase_key(input)
+        cache = fb.get(self.base_dir+"/"+self.cache_loc, fb_key)
+        if cache == None:
+            return []
+        else:
+            for item in cache:   #ie cache is list of dicts {'input':inStr,'value':cached_items_list}
+                if item['input'] == input:
+                    return item['value']
+        
+        return []
+    
+    def add_to_cache(self,input,value):
+        """
+        Adds the input and value to the cache
+        This is used for adding documents to the vector store
+
+        Returns:
+            The True if item in cache was updated and false if a new item was added
+
+        """
+        retVal=False
+        fb_key=fbh.convert_to_firebase_key(input)
+        cache = fb.get(self.base_dir+"/"+self.cache_loc, fb_key)
+        if cache is None:
+            cache = []
+        else:
+            for item in cache:   #ie cache is list of dicts {'input':inStr,'value':cached_items_list}
+                if item['input'] == input:
+                    item['value']=value
+                    retVal=True
+        cache.append({'input':input,'value':value})
+        fb.patch(self.base_dir+"/"+self.cache_loc, {fb_key:cache})
+        return retVal
+    
+       
+    

src/baseInfra/dropbox_handler.py

@@ -0,0 +1,152 @@
+import datetime
+import time
+import dropbox
+from dropbox.files import WriteMode
+from dropbox.exceptions import ApiError, AuthError
+import sys,os
+import baseInfra.firebase_handler as fbh
+
+TOKEN=fbh.fb_get("d2_accesstoken")
+APP_KEY=os.environ['DROPBOX_APP_KEY']
+APP_SECRET=os.environ['DROPBOX_APP_SECRET']
+REFRESH_TOKEN=fbh.fb_get("d2_refreshtoken")
+
+#os.environ['DROP_DIR2']="C:/dockers/chroma/chroma1/"
+#os.environ['APP_PATH']="/"
+#print("token::",TOKEN)
+
+with dropbox.Dropbox(oauth2_access_token=TOKEN,app_key=APP_KEY,app_secret=APP_SECRET,oauth2_refresh_token=REFRESH_TOKEN) as dbx: #,app_key=APP_KEY,app_secret=APP_SECRET,oauth2_refresh_token=REFRESH_TOKEN) as dbx:
+    # Check that the access token is valid
+    try:
+        dbx.users_get_current_account()
+        if (TOKEN != dbx._oauth2_access_token):
+            fbh.fb_update("d2_accesstoken",dbx._oauth2_access_token)
+            TOKEN=dbx._oauth2_access_token
+        print("dropbox connection ok,",dbx._oauth2_access_token)
+        print(dbx._oauth2_refresh_token)
+    except AuthError:
+        try:
+            dbx.check_and_refresh_access_token()
+            fbh.fb_update("d2_accesstoken",dbx._oauth2_access_token)
+            print("dropbox connection refreshed and updated",dbx._oauth2_access_token)
+            print(dbx._oauth2_refresh_token)
+        except Exception:
+            sys.exit("ERROR: Invalid access token; try re-generating an "
+                "access token from the app console on the web.")
+
+def normalizeFilename(filename):
+    while '//' in filename:
+        filename = filename.replace('//', '/')
+    return filename
+
+def getDropboxFilename(localFilename):
+    """ localFilename is $DROP_DIR2/<subpath>/<filename>"""
+    """ dropboxFilename is $APP_PATH/<subpath>/<filename"""
+    #if not localFilename.startswith(os.environ['DROP_DIR2']):
+    #    localFilename=os.environ['DROP_DIR2']+localFilename
+    localFilename=normalizeFilename(localFilename)
+    return normalizeFilename(localFilename.replace(os.environ['DROP_DIR2'],"/",1).replace("/",os.environ['APP_PATH'],1))
+
+def getLocalFilename(dropboxFilename):
+    """ localFilename is $DROP_DIR2/<subpath>/<filename>"""
+    """ dropboxFilename is $APP_PATH/<subpath>/<filename"""
+    #if not dropboxFilename.startswith(os.environ['APP_PATH']):
+    #    dropboxFilename=os.environ['APP_PATH']+dropboxFilename
+    dropboxFilename=normalizeFilename(dropboxFilename)
+    return normalizeFilename(dropboxFilename.replace(os.environ['APP_PATH'],"/",1).replace("/",os.environ['DROP_DIR2'],1))
+
+def backupFile(localFilename):
+    """Upload a file.
+    Return the request response, or None in case of error.
+    This will also create directory on dropbox if needed
+    """
+    global TOKEN
+    localFilename=normalizeFilename(localFilename) 
+    dropboxFilename=getDropboxFilename(localFilename)
+    print("backing file ",localFilename," to ",dropboxFilename)
+    mode = dropbox.files.WriteMode.overwrite
+    mtime = os.path.getmtime(localFilename)
+    with open(localFilename, 'rb') as f:
+        data = f.read()
+    try:
+        res = dbx.files_upload(
+            data, dropboxFilename, mode,
+            client_modified=datetime.datetime(*time.gmtime(mtime)[:6]),
+            mute=True)
+        if (TOKEN != dbx._oauth2_access_token):
+            fbh.fb_update("d2_accesstoken",dbx._oauth2_access_token)
+            TOKEN=dbx._oauth2_access_token
+            print(dbx._oauth2_refresh_token)
+    except dropbox.exceptions.ApiError as err:
+        print('*** API error', err)
+        return None
+    print('uploaded as', res.name.encode('utf8'))
+    return res
+
+def restoreFile(dropboxFilename):
+    """Download a file.
+    Return the bytes of the file, or None if it doesn't exist.
+    Will create dir+subdirs if possible
+    """
+    global TOKEN
+    dropboxFilename=normalizeFilename(dropboxFilename)
+    localFilename=getLocalFilename(dropboxFilename)
+    print("restoring file ",localFilename," from ",dropboxFilename)
+    try:
+        md, res = dbx.files_download(dropboxFilename)
+        if (TOKEN != dbx._oauth2_access_token):
+            fbh.fb_update("d2_accesstoken",dbx._oauth2_access_token)
+            TOKEN=dbx._oauth2_access_token
+            print(dbx._oauth2_refresh_token)
+    except dropbox.exceptions.HttpError as err:
+        print('*** HTTP error', err)
+        return None
+    data = res.content
+    print(len(data), 'bytes; md:', md)
+    localdir=os.path.dirname(localFilename)
+    if not os.path.exists(localdir):
+        os.makedirs(localdir)
+    with open(localFilename, 'wb') as f:
+        f.write(data)
+    return data
+
+def backupFolder(localFolder):
+    """ list all files in folder and subfolder and upload them"""
+    print("backup folder called for ",localFolder)
+    if not localFolder.startswith(os.environ['DROP_DIR2']):
+        localFolder=os.environ['DROP_DIR2']+localFolder
+    filenames=[]
+    for (root,dirs,files) in os.walk(localFolder, topdown=True):
+        print(root)
+        for filename in files:
+            filenames.append(root+"/"+filename)
+            print(root+"/"+filename)
+            backupFile(root+"/"+filename)
+
+
+def restoreFolder(dropboxFolder):
+    """ list all files in dropbox folder and subfolders and restore them"""
+    global TOKEN
+    if not dropboxFolder.startswith(os.environ['APP_PATH']):
+        dropboxFolder=os.environ['APP_PATH']+dropboxFolder
+    try:
+        res=dbx.files_list_folder(dropboxFolder)
+        if (TOKEN != dbx._oauth2_access_token):
+            fbh.fb_update("d2_accesstoken",dbx._oauth2_access_token)
+            TOKEN=dbx._oauth2_access_token
+            print(dbx._oauth2_refresh_token)
+    except dropbox.exceptions.ApiError as err:
+        print('Folder listing failed for', dropboxFolder, '-- assumed empty:', err)
+        return
+    except dropbox.exceptions.AuthError as err1:
+        print('Folder listing failed for', dropboxFolder, '-- assumed empty:', err1)
+        return
+    for entry in res.entries:
+        if (isinstance(entry, dropbox.files.FileMetadata)):
+            restoreFile(entry.path_display)
+        else:
+            try:
+                restoreFolder(entry.path_display)
+            except Exception:
+                print("Error restoring folder,",entry.path_display)
+        print(entry.path_display)

src/baseInfra/firebase_handler.py

@@ -0,0 +1,97 @@
+from dotenv import load_dotenv
+load_dotenv()
+from firebase import firebase
+import os
+
+secret=os.environ['FIREBASE_TOKEN'] 
+user_id="db"   #this file is for dropbox token management through firebase only
+auth=firebase.FirebaseAuthentication(secret=secret,email="anubhav77@gmail.com",extra={"id":user_id})
+fb_url="https://device-1a455.firebaseio.com"
+fb= firebase.FirebaseApplication(fb_url,auth)
+base_dir="/users/"+user_id
+
+def fb_get(item):
+    """ item need to be in format d2_refreshtoken same name will be present in the firebase
+        d2 means second dropbox instance or app.
+    """
+    return fb.get(base_dir,item)
+
+def fb_update(item,value):
+    """ item need to be in format d2_refreshtoken same name will be present in the firebase
+        d2 means second dropbox instance or app.
+    """
+    return fb.patch(base_dir,{item:value})
+
+import re
+
+def convert_to_firebase_key(inStr):
+  """Converts a string to a Firebase key.
+
+  Args:
+    inStr: The string to convert.
+
+  Returns:
+    A Firebase key, or None if the inStr is not a valid Firebase key.
+  """
+
+  # Firebase keys must be between 3 and 128 characters long.
+  # They must start with a letter, number, or underscore.
+  # They can only contain letters, numbers, underscores, hyphens, and periods.
+  # They cannot contain spaces or other special characters.
+
+  # Convert the inStr to lowercase.
+  outStr = inStr.lower()
+
+  # Replace all spaces with underscores.
+  outStr = outStr.replace(' ', '_')
+
+  # Remove any leading or trailing underscores.
+  outStr = outStr.lstrip('_').rstrip('_')
+
+  # Remove any character not matching firebase_key_Regex.
+  outStr=re.sub(r'[^\w\-.]', '', outStr)
+
+
+  firebase_key_regex = re.compile(r'^[a-zA-Z0-9_][a-zA-Z0-9-_.]*$')
+  # The following string values are not allowed for Firebase keys:
+  # - "firebase"
+  # - "google"
+  # - "android"
+  # - "ios"
+  # - "web"
+  # - "console"
+  # - "auth"
+  # - "database"
+  # - "storage"
+  # - "hosting"
+  # - "functions"
+  # - "firestore"
+  # - "realtime-database"
+  # - "remote-config"
+  # - "analytics"
+  # - "crashlytics"
+  # - "performance-monitoring"
+  # - "test-lab"
+  # - "cloud-messaging"
+  # - "dynamic-links"
+  # - "identity-toolkit"
+  # - "cloud-functions"
+  # - "cloud-firestore"
+  # - "cloud-storage"
+  # - "cloud-hosting"
+  # - "cloud-functions-v2"
+  # - "cloud-firestore-v2"
+  # - "cloud-storage-v2"
+  # - "cloud-hosting-v2"
+  # - "cloud-functions-v3"
+  # - "cloud-firestore-v3"
+  # - "cloud-storage-v3"
+  # - "cloud-hosting-v3"
+
+  if not firebase_key_regex.match(inStr) or inStr in ['firebase', 'google', 'android', 'ios', 'web', 'console', 'auth', 'database', 'storage', 'hosting', 'functions', 'firestore', 'realtime-database', 'remote-config', 'analytics', 'crashlytics', 'performance-monitoring', 'test-lab', 'cloud-messaging', 'dynamic-links', 'identity-toolkit', 'cloud-functions', 'cloud-firestore', 'cloud-storage', 'cloud-hosting', 'cloud-functions-v2', 'cloud-firestore-v2', 'cloud-storage-v2', 'cloud-hosting-v2', 'cloud-functions-v3', 'cloud-firestore-v3', 'cloud-storage-v3', 'cloud-hosting-v3']:
+    return "fb_"+outStr
+
+  if len(outStr) > 120:
+     outStr = "fbx_"+outStr[:120]
+
+  return outStr

src/indexer.py

@@ -0,0 +1,106 @@
+from langchain.vectorstores import Chroma
+from chromadb.api.fastapi import requests
+from langchain.schema import Document
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceBgeEmbeddings
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.chains.query_constructor.base import AttributeInfo
+from llm.llmFactory import LLMFactory
+
+model_name = "BAAI/bge-base-en"
+encode_kwargs = {'normalize_embeddings': True} # set True to compute cosine similarity
+
+embedding = HuggingFaceBgeEmbeddings(
+    model_name=model_name,
+    model_kwargs={'device': 'cpu'},  
+    encode_kwargs=encode_kwargs
+)
+
+persist_directory = 'db'
+docs = [
+    Document(
+        page_content="Complex, layered, rich red with dark fruit flavors",
+        metadata={"name":"Opus One", "year": 2018, "rating": 96, "grape": "Cabernet Sauvignon", "color":"red", "country":"USA"},
+    ),
+    Document(
+        page_content="Luxurious, sweet wine with flavors of honey, apricot, and peach",
+        metadata={"name":"Château d'Yquem", "year": 2015, "rating": 98, "grape": "Sémillon", "color":"white", "country":"France"},
+    ),
+    Document(
+        page_content="Full-bodied red with notes of black fruit and spice",
+        metadata={"name":"Penfolds Grange", "year": 2017, "rating": 97, "grape": "Shiraz", "color":"red", "country":"Australia"},
+    ),
+    Document(
+        page_content="Elegant, balanced red with herbal and berry nuances",
+        metadata={"name":"Sassicaia", "year": 2016, "rating": 95, "grape": "Cabernet Franc", "color":"red", "country":"Italy"},
+    ),
+    Document(
+        page_content="Highly sought-after Pinot Noir with red fruit and earthy notes",
+        metadata={"name":"Domaine de la Romanée-Conti", "year": 2018, "rating": 100, "grape": "Pinot Noir", "color":"red", "country":"France"},
+    ),
+    Document(
+        page_content="Crisp white with tropical fruit and citrus flavors",
+        metadata={"name":"Cloudy Bay", "year": 2021, "rating": 92, "grape": "Sauvignon Blanc", "color":"white", "country":"New Zealand"},
+    ),
+    Document(
+        page_content="Rich, complex Champagne with notes of brioche and citrus",
+        metadata={"name":"Krug Grande Cuvée", "year": 2010, "rating": 93, "grape": "Chardonnay blend", "color":"sparkling", "country":"New Zealand"},
+    ),
+    Document(
+        page_content="Intense, dark fruit flavors with hints of chocolate",
+        metadata={"name":"Caymus Special Selection", "year": 2018, "rating": 96, "grape": "Cabernet Sauvignon", "color":"red", "country":"USA"},
+    ),
+    Document(
+        page_content="Exotic, aromatic white with stone fruit and floral notes",
+        metadata={"name":"Jermann Vintage Tunina", "year": 2020, "rating": 91, "grape": "Sauvignon Blanc blend", "color":"white", "country":"Italy"},
+    ),
+]
+
+vectorstore = Chroma.from_documents(documents=docs,
+                                  embedding=embedding,
+                                  persist_directory=persist_directory)
+
+metadata_field_info = [
+    AttributeInfo(
+        name="grape",
+        description="The grape used to make the wine",
+        type="string or list[string]",
+    ),
+    AttributeInfo(
+        name="name",
+        description="The name of the wine",
+        type="string or list[string]",
+    ),
+    AttributeInfo(
+        name="color",
+        description="The color of the wine",
+        type="string or list[string]",
+    ),
+    AttributeInfo(
+        name="year",
+        description="The year the wine was released",
+        type="integer",
+    ),
+    AttributeInfo(
+        name="country",
+        description="The name of the country the wine comes from",
+        type="string",
+    ),
+    AttributeInfo(
+        name="rating", description="The Robert Parker rating for the wine 0-100", type="integer" #float
+    ),
+]
+document_content_description = "Brief description of the wine"
+lf=LLMFactory()
+llm=lf.get_llm("executor2")
+
+retriever = SelfQueryRetriever.from_llm(
+    llm,
+    vectorstore,
+    document_content_description,
+    metadata_field_info,
+    verbose=True
+)
+
+def getRelevantDocs(query:str):
+    return retriever.get_relevant_documents(query)

src/llm/hostedLLM.py

@@ -0,0 +1,92 @@
+from typing import Any, List, Mapping, Optional, Dict 
+from pydantic import Extra, Field #, root_validator, model_validator
+import os,json
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.base import LLM
+import requests
+
+
+class HostedLLM(LLM):
+    """
+       Hosted LLMs in huggingface spaces with fastAPI. Interface is primarily rest call with huggingface token
+
+       Attributes:
+       url: is the url of the endpoint 
+       http_method: which http_method need to be invoked [get or post]
+       model_name: which model is being hosted
+       temperature: temperature between 0 to 1
+       max_tokens: amount of output to generate, 512 by default
+       api_token: api_token to be passed for bearer authorization. Defaults to huggingface_api enviorment variable.
+       verbose: for extra logging
+    """
+    url: str = ""
+    http_method: Optional[str] = "post"
+    model_name: Optional[str] = "bard"
+    api_token: Optional[str] = os.environ["HUGGINGFACE_API"]
+    temperature: float = 0.7
+    max_tokens: int = 512
+    verbose: Optional[bool] = False
+    class Config:
+        extra = Extra.forbid
+
+    #@model_validator(mode="after")
+    #def validate_environment(cls, values: Dict) -> Dict:
+    #    if values["http_method"].strip() == "GET" or values["http_method"].strip() == "get":
+    #        values["http_method"]="get"
+    #    else:
+    #        values["http_method"]="post"
+    #    if values["api_token"] == "":
+    #        values["api_token"] = os.environ["HUGGINGFACE_API"]
+    #
+    #    return values
+    
+    @property
+    def _llm_type(self) -> str:
+        return "text2text-generation"
+    
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+    ) -> str:
+        if run_manager:
+            run_manager.on_text([prompt])
+        #messages={"messages":[{"role":"user","content":prompt}]}
+        prompt={"prompt":prompt}
+        headers = {
+            "Authorization": f"Bearer {self.api_token}",
+            "Content-Type": "application/json",
+        }
+        if(self.http_method=="post"):
+            response=requests.post(self.url,json=prompt,headers=headers)
+        else:
+            response=requests.get(self.url,json=prompt,headers=headers)
+        val=json.loads(response.text)['content']
+        if run_manager:
+            run_manager.on_llm_end(val)
+        return val
+    
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        """Get the identifying parameters."""
+        return {"name": self.model_name, "type": "hosted"}
+    
+    def extractJson(self,val:str) -> Any:
+        """Helper function to extract json from this LLMs output"""
+        #This is assuming the json is the first item within ````
+        #my super LLM will sometime send the json directly
+        try:
+            v3=val.replace("\n","").replace("\r","")
+            v4=json.loads(v3)
+        except:
+            v2=val.replace("```json","```").split("```")[1]
+            v3=v2.replace("\n","").replace("\r","")
+            v4=json.loads(v3)
+        return v4
+    
+    def extractPython(self,val:str) -> Any:
+        """Helper function to extract python from this LLMs output"""
+        #This is assuming the python is the first item within ````
+        v2=val.replace("```python","```").split("```")[1]
+        return v2

src/llm/llmFactory.py

@@ -0,0 +1,55 @@
+import logging
+from baseInfra.dbInterface import DbInterface
+from llm.hostedLLM import HostedLLM
+from llm.togetherLLM import TogetherLLM
+from llm.palmLLM import PalmLLM
+
+
+class LLMFactory:
+    """
+    Factory class for creating LLM objects.
+    """
+
+    def __init__(self):
+        """
+        Constructor for the LLMFactory class.
+
+        Args:
+            db_interface: The DBInterface object to use for getting LLM configs.
+        """
+        self._db_interface = DbInterface() 
+
+    def get_llm(self, llm_path: str) -> object:
+        """
+        Gets an LLM object of the specified type.
+
+        Args:
+            llm_path: The path to the LLM config.
+
+        Returns:
+            The LLM object.
+        """
+        logger = logging.getLogger(__name__)
+        try:
+            config = self._db_interface.get_config(llm_path)
+            logger.debug(llm_path)
+            logger.debug(config)
+            llm_type = config["llm_type"]
+            llm_config=config["llm_config"]
+        except Exception as ex:
+            logger.exception("Exception in getLLM")
+            logger.exception(ex)
+            config={}
+            llm_type=""
+            llm_config={}
+
+        if llm_type == "hostedLLM":
+            return HostedLLM(**llm_config)
+        elif llm_type == "togetherLLM":
+            return TogetherLLM(**llm_config)
+        elif llm_type == "palmLLM":
+            return PalmLLM(**llm_config)
+        else:
+            logger.error(f"Invalid LLM type: {llm_type}")
+            raise ValueError(f"Invalid LLM type: {llm_type}")
+

src/llm/palmLLM.py

@@ -0,0 +1,93 @@
+from typing import Any, List, Mapping, Optional, Dict 
+from pydantic import Extra, Field #, root_validator, model_validator
+import os,json
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.base import LLM
+import google.generativeai as palm
+from google.generativeai import types
+import ast
+#from langchain.llms import GooglePalm
+import requests
+
+
+class PalmLLM(LLM):
+    
+    model_name: str = "text-bison-001"
+    temperature: float = 0
+    max_tokens: int = 2048
+    stop: Optional[List] = []
+    prev_prompt: Optional[str]=""
+    prev_stop: Optional[str]=""
+    prev_run_manager:Optional[Any]=None
+
+    def __init__(
+        self,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        palm.configure()
+        #self.model = palm.Text2Text(self.model_name)
+
+    @property
+    def _llm_type(self) -> str:
+        return "text2text-generation"
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+    ) -> str:
+        self.prev_prompt=prompt
+        self.prev_stop=stop
+        self.prev_run_manager=run_manager
+        if stop == None:
+            stop=self.stop
+        text=palm.generate_text(prompt=prompt,stop_sequences=self.stop,
+                            temperature=self.temperature, max_output_tokens=self.max_tokens,
+                            safety_settings=[{"category":0,"threshold":4},
+                                             {"category":1,"threshold":4},
+                                             {"category":2,"threshold":4},
+                                             {"category":3,"threshold":4},
+                                             {"category":4,"threshold":4},
+                                             {"category":5,"threshold":4},
+                                             {"category":6, "threshold":4}]
+                        )
+        print("Response from palm",text)
+        val=text.result
+        if run_manager:
+            run_manager.on_llm_end(val)
+        return val
+    
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        """Get the identifying parameters."""
+        return {"name": self.model_name, "type": "palm"} 
+    
+    def extractJson(self,val:str) -> Any:
+        """Helper function to extract json from this LLMs output"""
+        #This is assuming the json is the first item within ````
+        # palm is responding always with ```json and ending with ```, however sometimes response is not complete
+        # in case trailing ``` is not seen, we will call generation again with prev_prompt and result appended to it
+        try:
+            count=0
+            while val.startswith("```json") and not val.endswith("```") and count<7:
+                val=self._call(prompt=self.prev_prompt+" "+val,stop=self.prev_stop,run_manager=self.prev_run_manager)
+                count+=1
+            v2=val.replace("```json","```").split("```")[1]
+            #v3=v2.replace("\n","").replace("\r","").replace("'","\"")
+            v3=json.dumps(ast.literal_eval(v2))
+            v4=json.loads(v3)
+        except:
+            v2=val.replace("\n","").replace("\r","")
+            v3=json.dumps(ast.literal_eval(val))
+            #v3=v2.replace("'","\"")
+            v4=json.loads(v3)
+            #v4=json.loads(v2)
+        return v4
+    
+    def extractPython(self,val:str) -> Any:
+        """Helper function to extract python from this LLMs output"""
+        #This is assuming the python is the first item within ````
+        v2=val.replace("```python","```").split("```")[1]
+        return v2

src/llm/togetherLLM.py

@@ -0,0 +1,72 @@
+import together
+import os
+import logging,json
+from typing import Any, Dict, List, Mapping, Optional
+
+from pydantic import Extra, Field #, root_validator, model_validator
+
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.base import LLM
+from langchain.llms.utils import enforce_stop_tokens
+from langchain.utils import get_from_dict_or_env
+
+class TogetherLLM(LLM):
+    """Together large language models."""
+
+    model_name: str = "togethercomputer/llama-2-70b-chat"
+    """model endpoint to use"""
+
+    together_api_key: str = os.environ["TOGETHER_API_KEY"]
+    """Together API key"""
+
+    temperature: float = 0.7
+    """What sampling temperature to use."""
+
+    max_tokens: int = 512
+    """The maximum number of tokens to generate in the completion."""
+
+    class Config:
+        extra = Extra.forbid
+
+    #@model_validator(mode="after")
+    #def validate_environment(cls, values: Dict) -> Dict:
+    #    """Validate that the API key is set."""
+    #    api_key = get_from_dict_or_env(
+    #        values, "together_api_key", "TOGETHER_API_KEY"
+    #    )
+    #    values["together_api_key"] = api_key
+    #    return values
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of LLM."""
+        return "together"
+
+    def _call(
+        self,
+        prompt: str,
+        **kwargs: Any,
+    ) -> str:
+        """Call to Together endpoint."""
+        together.api_key = self.together_api_key
+        output = together.Complete.create(prompt,
+                                          model=self.model_name,
+                                          max_tokens=self.max_tokens,
+                                          temperature=self.temperature,
+                                          )
+        text = output['output']['choices'][0]['text']
+        return text
+
+    def extractJson(self,val:str) -> Any:
+        """Helper function to extract json from this LLMs output"""
+        #This is assuming the json is the first item within ````
+        v2=val.replace("```json","```").split("```")[1]
+        v3=v2.replace("\n","").replace("\r","")
+        v4=json.loads(v3)
+        return v4
+    
+    def extractPython(self,val:str) -> Any:
+        """Helper function to extract python from this LLMs output"""
+        #This is assuming the python is the first item within ````
+        v2=val.replace("```python","```").split("```")[1]
+        return v2