Initial commit

.github/workflows/dockerhub.yaml

@@ -0,0 +1,39 @@
+name: Publish ConversAI Docker image
+
+on:
+  push:
+    branches: [ dev ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: techconsp/tcp_frwjeiqhpyl9ty53cyfg2jtpbhwwzl_co
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -0,0 +1,148 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Machine Learning and Speech Libraries
+# TensorFlow
+*.ckpt*
+*.pbtxt
+*.tfevents*
+# PyTorch
+*.pt
+# Keras
+*.h5
+# Scikit-learn
+*.pkl
+# Speech Recognition
+*.wav
+*.mp3
+.idea/
+logs
+images
+resources

Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY . /app
+
+RUN chmod -R 777 /app
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y \
+        build-essential \
+        git \
+        cmake \
+        poppler-utils \
+        ffmpeg \
+        libsm6 \
+        libxext6 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir nltk && \
+    mkdir -p /app/nltk_data && \
+    chmod -R 777 /app/nltk_data && \
+    python -m nltk.downloader -d /app/nltk_data all
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--timeout-keep-alive", "300", "--timeout-graceful-shutdown", "600"]
+

app.py

@@ -0,0 +1,47 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import uvicorn
+from fastapi import FastAPI, Depends
+from fastapi.middleware.cors import CORSMiddleware
+from src import logging as logger
+from src.api.conversai_api import conversai_api_router
+from src.api.speech_api import speech_translator_router
+from src.api.user_management_api import user_management_api_router
+from src.utils.utils import load_ini_config
+from src.api.analytics_api import analytic_endpoints_router
+from src.api.jwt_bearer import access_check_bearer
+
+logger.info("---------------------------> Starting FastAPI Server <---------------------------")
+
+config = load_ini_config("config.ini")
+app = FastAPI(docs_url=config.get('fastapi_config', 'docs_url'), redoc_url=config.get('fastapi_config', 'redoc_url'),
+              openapi_url=config.get('fastapi_config', 'openapi_url')
+              )
+
+PROTECTED = [Depends(access_check_bearer)]
+
+logger.info("---------------------------> FastAPI Server Started <---------------------------")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allow access from all sources
+    allow_credentials=True,
+    allow_methods=["*"],  # Allow all HTTP methods
+    allow_headers=["*"],  # Allow all request headers
+)
+app.include_router(user_management_api_router, prefix="/conversai")
+logger.info("---------------------------> User Management API Started <---------------------------")
+
+app.include_router(conversai_api_router, prefix="/conversai")
+logger.info("---------------------------> ConversAI API Started <---------------------------")
+
+app.include_router(speech_translator_router, prefix="/conversai", dependencies=PROTECTED)
+logger.info("---------------------------> Speech Translator API Started <---------------------------")
+
+app.include_router(analytic_endpoints_router, prefix="/conversai", dependencies=PROTECTED)
+logger.info("---------------------------> Analytics API Started <---------------------------")
+
+if __name__ == '__main__':
+    uvicorn.run(app, port=int(config.get('fastapi_config', 'port')), host=config.get('fastapi_config', 'host'),
+                timeout_keep_alive=300, timeout_graceful_shutdown=600)

config.ini

@@ -0,0 +1,39 @@
+[default]
+chatbot_name = ConversAI
+chatbot_prefix = convai
+
+[data_analyzer]
+groq_llm_name = llama-3.1-8b-instant
+additional_query = .In case, you are to plot a chart, make sure the x-axis labels are 90 degree rotated.
+verbose = False
+
+[easy_ocr]
+model_path = resources/easyocr_model
+language = en
+gpu = True
+
+[speech_to_text]
+model_id = openai/whisper-large-v3
+max_new_tokens = 128
+chunks_length_s = 30
+batch_size = 16
+
+[supabase_chatbot_management]
+user_config_table = ConversAI_UserConfig
+chat_bot_table = ConversAI_ChatbotInfo
+
+[all_mini_l6_v2_vector_embedding]
+device = cpu
+normalize_embeddings = True
+
+
+[fastapi_config]
+host = 0.0.0.0
+port = 8000
+docs_url = /docs
+redoc_url = /redoc
+openapi_url = /openapi.json
+
+
+[oauth]
+redirect_to : https://convers-ai-test.vercel.app/home/

examples/easy_ocr_example.py

@@ -0,0 +1,12 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import cv2
+
+from src.services.ocr.easy_ocr.easy_ocr_ import EasyOCR_
+
+if __name__ == '__main__':
+    image = cv2.imread("images/img.png")
+    ocr = EasyOCR_()
+    print(ocr.read_text(image))

examples/url_text_extraction_example.py

@@ -0,0 +1,10 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from src.services.website_url.text_extraction_urls import extract_text_from_url
+
+if __name__ == '__main__':
+    website = "https://huggingface.co/BAAI/bge-m3"
+    extracted_text = extract_text_from_url(website)
+    print(extracted_text)

logging_config.yaml

@@ -0,0 +1,27 @@
+version: 1
+disable_existing_loggers: False
+formatters:
+  standard:
+    format: '[%(asctime)s: %(levelname)s: %(module)s: %(message)s]'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: standard
+    stream: ext://sys.stdout
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    level: INFO
+    formatter: standard
+    filename: logs/application.log
+    when: midnight
+    interval: 1
+    backupCount: 30
+loggers:
+  __main__:
+    level: DEBUG
+    handlers: [console, file]
+    propagate: no
+root:
+  level: DEBUG
+  handlers: [console, file]

params.yaml

@@ -0,0 +1,27 @@
+prompt: |
+  INSTRUCTIONS:
+  =====================================
+  ### Role
+      **Primary Function**: You are an AI chatbot designed to provide accurate and efficient assistance to users based on provided context data. Your responses must be reliable, friendly, and directly address user inquiries or issues. Always clarify any unclear questions, and conclude responses positively.
+  ### Constraints
+      1. **No Data Disclosure**: Never reveal access to training data or any context explicitly.
+      2. **Maintaining Focus**: Politely redirect any off-topic conversations back to relevant issues without breaking character.
+      3. **Exclusive Reliance on Context Data**: Base all answers strictly on the provided context data. If the context doesn’t cover the query, use a fallback response. Always maintain a third-person perspective.
+      4. **Restrictive Role Focus**: Do not engage in tasks or answer questions unrelated to your role or context data.
+  Ensure all instructions are strictly followed. Responses must be meaningful and concise, within 512 words. Make sure the user is always happy and satisfied with the outputs you return.
+  CONTEXT:
+  =====================================
+  {context}
+  ======================================
+  QUESTION:
+  =====================================
+  {question}
+  CHAT HISTORY:
+  =====================================
+  {chatHistory}
+  NOTE: Generate responses directly without using phrases like "Response:" or "Answer:". Do not mention the use of extracted context or provide unnecessary details.
+
+follow_up_prompt: |
+  You are an expert chatbot at framing follow up questions using some given text such that their answers can be found in the text itself and have been given the task of doing the same. Make sure that the questions are good quality and not too long in length. Frame appropriate and meaningful questions out of the given text and DO NOT mention the usage of any text in the questions. Also, if no the given text says NO CONTEXT FOUND, please return an empty string for each question asked.
+  \n{format_instructions}
+  \n{context}

requirements.txt

@@ -0,0 +1,99 @@
+accelerate==0.33.0
+annotated-types==0.7.0
+anyio==4.4.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+dnspython==2.6.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
+fastapi==0.111.1
+fastapi-cli==0.0.4
+filelock==3.15.4
+fsspec==2024.6.1
+gTTS==2.5.2
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.24.5
+idna==3.7
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.6.20
+nvidia-nvtx-cu12==12.1.105
+packaging==24.1
+psutil==6.0.0
+pydantic==2.8.2
+pydantic_core==2.20.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+PyYAML==6.0.1
+regex==2024.7.24
+requests==2.32.3
+rich==13.7.1
+safetensors==0.4.3
+scikit-build==0.18.0
+shellingham==1.5.4
+sniffio==1.3.1
+starlette==0.37.2
+sympy==1.13.1
+tokenizers==0.19.1
+tomli==2.0.1
+torch==2.4.0
+tqdm==4.66.4
+transformers @ git+https://github.com/huggingface/transformers.git@85a1269e19af022e04bc2aad82572cd5a9e8cdd9
+triton==3.0.0
+typer==0.12.3
+typing_extensions==4.12.2
+urllib3==2.2.2
+uvicorn==0.30.4
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==12.0
+bs4
+huggingface-hub
+fastembed
+nest_asyncio
+beautifulsoup4
+flashrank
+flashrank[listwise]
+PyMuPDF
+langchain
+langchain-community
+langchain-cohere
+langchain-huggingface
+langchain-qdrant
+langchain-groq
+lxml
+python-dotenv
+pillow
+pandas
+sentence-transformers
+supabase
+unstructured
+urllib3
+langsmith
+pandasai
+easyocr
+youtube-transcript-api
+pdf2image
+PyPDF2
+PyJWT
+replicate

src/__init__.py

@@ -0,0 +1,19 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+
+import logging.config
+import yaml
+import os
+
+if os.path.exists("logs"):
+    pass
+else:
+    os.makedirs("logs")
+
+log_config_path = os.path.join(os.getcwd(), "logging_config.yaml")
+with open(log_config_path, 'r') as file:
+    config = yaml.safe_load(file.read())
+
+logging.config.dictConfig(config)

src/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/api/analytics_api.py

@@ -0,0 +1,235 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta
+from dateutil.parser import isoparse
+from fastapi.routing import APIRouter
+from src.pipeline.conversai_analytic_pipeline import ConversAIAnalyticPipeline
+from fastapi import Request
+from src.utils.error_handling import create_success_response, raise_http_exception, \
+    success_response_user_management
+from src.models.apis_models import FeedbackRequest, DailyActiveEndUserRequest, AverageSessionInteractionRequest, \
+    TokenUsageRequest, UserSatisfactionRateRequest
+from src import logging as logger
+
+analytic_endpoints_router = APIRouter(tags=["Analytics Endpoints"])
+
+conversai_analytic_pipeline = ConversAIAnalyticPipeline()
+
+
+@analytic_endpoints_router.post("/daily_chat_count")
+async def daily_chat_count(
+        request: DailyActiveEndUserRequest):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> daily_chat_count API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        dates = [
+            isoparse(i["timestamp"]).date()
+            for i in response
+            if start_date <= isoparse(i["timestamp"]).date() <= end_date
+        ]
+
+        date_count = Counter(dates)
+
+        data = [{"date": date.isoformat(), "count": count} for date, count in date_count.items()]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> daily_chat_count API Response Success  for {vectorstore} <<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>> daily_chat_count API Response Failed for {vectorstore} {e}<<<")
+
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/daily_active_end_user")
+async def daily_active_end_user(
+        request: DailyActiveEndUserRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> daily_active_end_user API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        ip_by_date = defaultdict(set)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            ip_address = i["IpAddress"]
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                ip_by_date[date].add(ip_address)
+
+        data = [{"date": date.isoformat(), "terminal": len(ips)} for date, ips in ip_by_date.items() if len(ips) > 1]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> daily_active_end_user API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> daily_active_end_user API Response Failed for {vectorstore} {e}<<<")
+
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/average_session_interaction")
+async def average_session_interaction(
+        request: AverageSessionInteractionRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> average_session_interaction API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        total_messages_by_date = defaultdict(int)
+        unique_ips_by_date = defaultdict(set)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            ip_address = i["IpAddress"]
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                total_messages_by_date[date] += 1
+                unique_ips_by_date[date].add(ip_address)
+
+        data = []
+        for date in sorted(total_messages_by_date.keys()):
+            total_messages = total_messages_by_date[date]
+            unique_ips = len(unique_ips_by_date[date])
+            average_interactions = total_messages / unique_ips if unique_ips > 0 else 0
+            data.append({"date": date.isoformat(), "interactions": average_interactions})
+
+        response = create_success_response(code=200, data=dict(data=data))
+        logger.info(f">>> average_session_interaction API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> average_session_interaction API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/token_usages")
+async def token_usages(request: TokenUsageRequest):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> token_usages API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        response = conversai_analytic_pipeline.chat_history_table_(vectorstore=vectorstore)
+
+        token_usage_by_date = defaultdict(int)
+
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                response_token_count = i.get("ResponseTokenCount")
+                if response_token_count is not None:
+                    token_usage_by_date[date] += response_token_count
+
+        data = [{"date": date.isoformat(), "total_tokens": total_tokens} for date, total_tokens in
+                token_usage_by_date.items()]
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> token_usages API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>> token_usages API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/add_feedback")
+async def add_feedback(req: Request, request: FeedbackRequest):
+    feedback, user_id, vectorstore = request.feedback, request.user_id, request.vectorstore
+    try:
+        logger.info(f">>> add_feedback API Triggered by {request.vectorstore} <<<")
+
+        client_ip = req.client.host
+        city = conversai_analytic_pipeline.get_ip_info(client_ip)
+
+        conversai_analytic_pipeline.add_feedback_(feedback, user_id, city, client_ip, vectorstore)
+
+        response = success_response_user_management(code=200, message="Add Feedback Sucess")
+        logger.info(f">>> add_feedback API Response Success for {vectorstore} <<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>> add_feedback API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@analytic_endpoints_router.post("/user_satisfaction_rate")
+async def user_satisfaction_rate(
+        request: UserSatisfactionRateRequest
+):
+    start_date, end_date, vectorstore = request.start_date, request.end_date, request.vectorstore
+    logger.info(f">>> user_satisfaction_rate API Triggered by {vectorstore} <<<")
+    try:
+        if not start_date or not end_date:
+            end_date = datetime.now().astimezone().date()
+            start_date = end_date - timedelta(days=7)
+        else:
+            start_date = isoparse(start_date).date()
+            end_date = isoparse(end_date).date()
+
+        feedback_counts = defaultdict(lambda: {"like": 0, "dislike": 0})
+        response = conversai_analytic_pipeline.feedback_table_(vectorstore)
+        for i in response:
+            timestamp = isoparse(i["timestamp"])
+            if start_date <= timestamp.date() <= end_date:
+                date = timestamp.date()
+                feedback = i.get("feedback")
+                if feedback == "like":
+                    feedback_counts[date]["like"] += 1
+                elif feedback == "dislike":
+                    feedback_counts[date]["dislike"] += 1
+
+        data = []
+        for date in sorted(feedback_counts.keys()):
+            like_count = feedback_counts[date]["like"]
+            dislike_count = feedback_counts[date]["dislike"]
+            total_feedback = like_count + dislike_count
+            satisfaction_rate = (like_count / total_feedback * 100) if total_feedback > 0 else 0
+            data.append({"date": date.isoformat(), "rate": satisfaction_rate})
+
+        response = create_success_response(code=200, data=dict(output=data))
+        logger.info(f">>> user_satisfaction_rate API Response Success for {vectorstore} <<<")
+
+        return response
+    except Exception as e:
+        logger.info(f">>> user_satisfaction_rate API Response Failed for {vectorstore} {e}<<<")
+        raise_http_exception(500, "Internal Server Error")

src/api/conversai_api.py

@@ -0,0 +1,631 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+import io
+import json
+import os
+import string
+import tempfile
+import requests
+import pandas as pd
+from src import logging as logger
+from supabase import create_client
+from urllib.parse import urlparse
+
+from src.api.jwt_bearer import access_check_bearer
+from src.models.apis_models import *
+from fastapi.requests import Request
+from fastapi.routing import APIRouter
+from fastapi import UploadFile, File, HTTPException, Form, Depends
+from src.pipeline.conversai_pipeline import ConversAIPipeline
+from src.api.user_management_api import user_management
+from src.services.supabase.analytics.analytic_tables import track_usage
+from src.services.supabase.user_management.token_limit import token_limit_check
+from src.utils.error_handling import create_error_response, create_success_response, raise_http_exception
+from src.api.user_management_api import user_management as user_management_pipeline
+from src.utils.utils import get_ip_info, encode_to_base64, clean_text, decode_base64
+
+conversai_api_router = APIRouter(tags=["ConversAI"])
+
+supabase_client = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY"))
+supabase_client_ = supabase_client
+conversai_pipeline = ConversAIPipeline()
+
+
+@conversai_api_router.post("/add_text", dependencies=[Depends(access_check_bearer)])
+async def add_text(request: AddTextRequest):
+    logger.info(f">>>AddText API Triggered By {request.vectorstore}<<<")
+    try:
+        vectorstore, text = request.vectorstore, request.text
+        track_usage(vectorstore=vectorstore, endpoint="/add_text", supabase_client=supabase_client)
+        username, chat_bot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chat_bot_name,
+                                text=text)
+        text = clean_text(text)
+        if lim:
+            dct = {
+                "output": {"text": text},
+                "source": "Text",
+            }
+            num_token = len(text.translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="text", username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+
+            supa = supabase_client.table("ConversAI_ChatbotDataSources").insert(
+                {"username": username, "chatbotName": chat_bot_name, "dataSourceName": file_name,
+                 "numTokens": num_token, "sourceEndpoint": "/add_text",
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                  f"{file_name}_data.json")}).execute()
+
+            response = create_success_response(200, {"message": "Successfully added the text."})
+            logger.info(f">>>Text added successfully for {request.vectorstore}.<<<")
+
+            return response
+        else:
+            response = create_error_response(400,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in add_text: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/answer_query")
+async def answer_query(request: AnswerQueryRequest, req: Request):
+    logger.info(f">>>answer_query API Triggered By {request.vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=request.vectorstore, endpoint="/answer_query")
+        username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        ip_address = req.client.host
+        city = get_ip_info(ip_address)
+        output, followup_questions, source = conversai_pipeline.answer_query_(query=request.query,
+                                                                              vectorstore=request.vectorstore,
+                                                                              llm_model=request.llm_model)
+        supa = supabase_client.table("ConversAI_ChatHistory").insert(
+            {"username": username, "chatbotName": chatbot_name, "llmModel": request.llm_model,
+             "question": request.query, "response": output, "IpAddress": ip_address, "ResponseTokenCount": len(output),
+             "vectorstore": request.vectorstore, "City": city}).execute()
+
+        response = create_success_response(200, data={"output": output, "follow_up_questions": followup_questions,
+                                                      "source": source})
+        logger.info(f">>>Query answered successfully for {request.vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in answer_query: {e} for {request.vectorstore}.<<<")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+
+
+@conversai_api_router.post("/data_analyzer", dependencies=[Depends(access_check_bearer)])
+async def data_analyzer(query: str = Form(...), file: UploadFile = File(...)):
+    logger.info(f">>>data_analyzer API Triggered By {query}<<<")
+    try:
+        extension = file.filename.split(".")[-1]
+        if extension in ["xls", "xlsx", "xlsm", "xlsb"]:
+            df = pd.read_excel(io.BytesIO(await file.read()))
+        elif extension == "csv":
+            df = pd.read_csv(io.BytesIO(await file.read()))
+        else:
+            return {"output": "INVALID FILE TYPE"}
+
+        response = conversai_pipeline.data_analyzer(query=query, dataframe=df)
+        response = create_success_response(200, {"output": response})
+        logger.info(f">>>Data analyzed successfully for {query}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in data_analyzer: {e} for {query}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/get_links", dependencies=[Depends(access_check_bearer)])
+async def get_links(request: GetLinksRequest):
+    logger.info(f">>>get_links API Triggered By {request.url}<<<")
+    try:
+        response = conversai_pipeline.get_links_(url=request.url, timeout=30)
+        response = create_success_response(200, {"urls": response, "source": urlparse(request.url).netloc})
+        logger.info(f">>>Links fetched successfully for {request.url}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_links: {e} for {request.url}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/image_pdf_text_extraction", dependencies=[Depends(access_check_bearer)])
+async def image_pdf_text_extraction(vectorstore: str = Form(...)
+                                    , pdf: UploadFile = File(...)):
+    logger.info(f">>>image_pdf_text_extraction API Triggered By {pdf.filename}<<<")
+    try:
+        track_usage(vectorstore=vectorstore, endpoint="/image_pdf_text_extraction", supabase_client=supabase_client)
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        source = pdf.filename
+        pdf_bytes = await pdf.read()
+        response = conversai_pipeline.image_pdf_text_extraction_(image_pdf=pdf_bytes)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=response)
+
+        if lim:
+            dct = {
+                "output": response,
+                "source": source
+            }
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=source, username=username)
+            num_tokens = len(
+                " ".join([response[x] for x in response]).translate(str.maketrans('', '', string.punctuation)).split(
+                    " "))
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            supa = supabase_client.table("ConversAI_ChatbotDataSources").insert(
+                {"username": username,
+                 "chatbotName": chatbot_name,
+                 "dataSourceName": file_name,
+                 "numTokens": num_tokens,
+                 "sourceEndpoint": "/image_pdf_text_extraction",
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                  f"{file_name}_data.json")}).execute()
+
+            response = create_success_response(200,
+                                               {"source": pdf.filename, "message": "Successfully extracted the text."})
+            logger.info(f">>>Text extracted successfully for {pdf.filename}.<<<")
+            return response
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of PDF or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in image_pdf_text_extraction: {e} for {pdf.filename}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/text_pdf_extraction", dependencies=[Depends(access_check_bearer)])
+async def text_pdf_extraction(vectorstore: str = Form(...)
+                              , pdf: UploadFile = File(...)):
+    logger.info(f">>>text_pdf_extraction API Triggered By {pdf.filename}<<<")
+    try:
+        track_usage(vectorstore=vectorstore, endpoint="/text_pdf_extraction", supabase_client=supabase_client)
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        source = pdf.filename
+        pdf = await pdf.read()
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+            temp_file.write(pdf)
+            temp_file_path = temp_file.name
+
+        response = conversai_pipeline.text_pdf_extraction_(pdf=temp_file_path)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=response)
+        os.remove(temp_file_path)
+        if lim:
+            dct = {
+                "output": response,
+                "source": source
+            }
+            numTokens = len(
+                " ".join([response[x] for x in response]).translate(str.maketrans('', '', string.punctuation)).split(
+                    " "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=source, username=username)
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            response = (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": numTokens,
+                         "sourceEndpoint": "/text_pdf_extraction",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"source": source, "message": "Successfully extracted the text."})
+            logger.info(f">>>Text extracted successfully for {source}.<<<")
+            return response
+
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of PDF or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in text_pdf_extraction: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/youtube_transcript", dependencies=[Depends(access_check_bearer)])
+async def youtube_transcript(request: YoutubeTranscriptRequest):
+    vectorstore, urls = request.vectorstore, request.urls
+    logger.info(f">>>youtube_transcript API Triggered By {urls}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/youtube_transcript")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+
+        response = conversai_pipeline.youtube_transcript_(url=urls)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=response)
+        if lim:
+
+            dct = {
+                "output": response,
+                "source": "www.youtube.com"
+            }
+            num_tokens = len(
+                " ".join([response[x] for x in response]).translate(str.maketrans('', '', string.punctuation)).split(
+                    " "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="youtube", username=username)
+            response = supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            response = (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_tokens,
+                         "sourceEndpoint": "/youtube_transcript",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"message": "Successfully fetched the youtube transcript."})
+            logger.info(f">>>Youtube transcript fetched successfully for {urls}.<<<")
+            return response
+        else:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in youtube_transcript: {e} for {urls}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/website_url_text_extraction", dependencies=[Depends(access_check_bearer)])
+async def add_website(request: AddWebsiteRequest):
+    vectorstore, website_urls, source = request.vectorstore, request.website_urls, request.source
+
+    logger.info(f">>>website_url_text_extraction API Triggered By {request.website_urls}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/fetch_text/urls")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+
+        text = conversai_pipeline.website_url_text_extraction_list_(urls=website_urls)
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chatbot_name,
+                                text=text)
+        if not lim:
+            response = create_error_response(402,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+        else:
+            dct = {
+                "output": text,
+                "source": source
+            }
+            num_tokens = len(
+                " ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name=urlparse(source).netloc,
+                                                                         username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chatbot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_tokens,
+                         "sourceEndpoint": "/fetch_text/urls",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+            response = create_success_response(200, {"message": "Successfully fetched the website text."})
+            logger.info(f">>>Website text extracted successfully for {request.website_urls}.<<<")
+            return response
+    except Exception as e:
+        logger.error(f">>>Error in website_url_text_extraction: {e} for {request.website_urls}.<<<")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+
+
+@conversai_api_router.get("/get_current_count", dependencies=[Depends(access_check_bearer)])
+async def get_count(vectorstore: str):
+    logger.info(f">>>get_current_count API Triggered By {vectorstore}<<<")
+    try:
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        current_count = user_management_pipeline.get_current_count_(username)
+
+        response = create_success_response(200, {"current_count": current_count})
+        logger.info(f">>>Current count fetched successfully for {vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_current_count: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/list_chatbots", dependencies=[Depends(access_check_bearer)])
+async def list_chatbots(request: ListChatbotsRequest):
+    logger.info(f">>>list_chatbots API Triggered By {request.username}<<<")
+    try:
+        chatbots = user_management.list_tables(username=request.username)
+        response = create_success_response(200, {"chatbots": chatbots})
+        logger.info(f">>>Chatbots listed successfully for {request.username}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in list_chatbots: {e} for {request.username}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/get_chat_history", dependencies=[Depends(access_check_bearer)])
+async def chat_history(request: GetChatHistoryRequest):
+    logger.info(f">>>get_chat_history API Triggered By {request.vectorstore}<<<")
+    try:
+        _, username, chatbotName = request.vectorstore.split("$", 2)
+
+        history = supabase_client.table("ConversAI_ChatHistory").select(
+            "timestamp", "question", "response"
+        ).eq("username", username).eq("chatbotName", chatbotName).execute().data
+
+        response = create_success_response(200, {"history": history})
+        logger.info(f">>>Chat history fetched successfully for {request.vectorstore}.<<<")
+        return response
+
+
+    except IndexError:
+        logger.warning(f"Chat history not found for {request.vectorstore}")
+        return create_error_response(404, "Chat history not found for the given chatbot.")
+
+    except Exception as e:
+        logger.error(f">>>Error in get_chat_history: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/delete_chatbot", dependencies=[Depends(access_check_bearer)])
+async def delete_chatbot(request: DeleteChatbotRequest):
+    logger.info(f">>>delete_chatbot API Triggered By {request.vectorstore}<<<")
+    try:
+        username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        supabase_client.table('ConversAI_ChatbotInfo').delete().eq('user_id', username).eq('chatbotname',
+                                                                                           chatbot_name).execute()
+        all_sources = supabase_client.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq(
+            "chatbotName", chatbot_name).execute().data
+        all_sources = [x["sourceContentURL"].split("/")[-1] for x in all_sources]
+        supabase_client.table("ConversAI_ChatbotDataSources").delete().eq("username", username).eq("chatbotName",
+                                                                                                   chatbot_name).execute()
+        supabase_client.table("ConversAI_ActivityLog").update({"isActive": False}).eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute()
+        supabase_client.table("ConversAI_ChatHistory").update({"isActive": False}).eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute()
+        for source in all_sources:
+            supabase_client.table("ConversAI_Chatbot")
+            supabase_client.storage.from_("ConversAI").remove(source)
+        user_management.delete_table(table_name=chatbot_name)
+        user_management.delete_qdrant_cluster(vectorstorename=request.vectorstore)
+        response = create_success_response(200, {"message": "Chatbot deleted successfully"})
+        logger.info(f">>>Chatbot deleted successfully for {request.vectorstore}.<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in delete_chatbot: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/add_qa_pair", dependencies=[Depends(access_check_bearer)])
+async def add_qa_pair(request: AddQAPairRequest):
+    logger.info(f">>>add_qa_pair API Triggered By {request.vectorstore}<<<")
+    try:
+        vectorstore, question, answer = request.vectorstore, request.question, request.answer
+        track_usage(vectorstore=vectorstore, endpoint="/add_qa_pair", supabase_client=supabase_client)
+        username, chat_bot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+        normal_text = f"\nQUESTION: {question}\nANSWER: {answer}\n"
+        lim = token_limit_check(supabase_client=supabase_client, username=username, chatbot_name=chat_bot_name,
+                                text=normal_text)
+        if lim:
+            dct = {
+                "output": {"text": normal_text},
+                "source": "QA Pair",
+            }
+            num_token = len(normal_text.translate(str.maketrans('', '', string.punctuation)).split(" "))
+            dct = json.dumps(dct, indent=1).encode("utf-8", errors="replace")
+            file_name = user_management_pipeline.create_data_source_name(source_name="qa_pair", username=username)
+            supabase_client.storage.from_("ConversAI").upload(file=dct, path=f"{file_name}_data.json")
+            (
+                supabase_client.table("ConversAI_ChatbotDataSources")
+                .insert({"username": username,
+                         "chatbotName": chat_bot_name,
+                         "dataSourceName": file_name,
+                         "numTokens": num_token,
+                         "sourceEndpoint": "/add_qa_pair",
+                         "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                          f"{file_name}_data.json")})
+                .execute()
+            )
+
+            response = create_success_response(200, {"message": "Successfully added the qa pair."})
+            logger.info(f">>>QA Pair added successfully for {request.vectorstore}.<<<")
+
+            return response
+        else:
+            response = create_error_response(400,
+                                             "Exceeding limits, please try with a smaller chunks of information or subscribe to our premium plan.")
+            return response
+
+    except Exception as e:
+        logger.error(f">>>Error in add_qa_pair: {e} for {request.vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/load_edited_json", dependencies=[Depends(access_check_bearer)])
+async def load_edited_json(request: LoadEditedJson):
+    vectorstore, data_source_name, source_endpoint, json_data = request.vectorstore, request.data_source_name, request.source_endpoint, request.json_data
+    username, chatbot_name = request.vectorstore.split("$")[1], request.vectorstore.split("$")[2]
+
+    logger.info(f">>>loadEditedJson API Triggered By {request.vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=request.vectorstore,
+                    endpoint="/load_edited_json")
+        json_data = decode_base64(request.json_data)
+        json_data = json.dumps(json_data, indent=1).encode("utf-8", errors="replace")
+        file_name = user_management_pipeline.create_data_source_name(source_name=data_source_name,
+                                                                     username=username)
+        response = supabase_client.storage.from_("ConversAI").upload(file=json_data, path=f"{file_name}_data.json")
+        response = (
+            supabase_client.table("ConversAI_ChatbotDataSources")
+            .insert({"username": username,
+                     "chatbotName": chatbot_name,
+                     "dataSourceName": file_name,
+                     "sourceEndpoint": source_endpoint,
+                     "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"],
+                                                      f"{file_name}_data.json")})
+            .execute()
+        )
+        response = create_success_response(200, {"output": "Successfully loaded the edited json."})
+        logger.info(f">>>Edited json loaded successfully for {vectorstore}.<<<")
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in loadEditedJson: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/list_chatbot_sources", dependencies=[Depends(access_check_bearer)])
+async def list_chatbot_sources(vectorstore: str):
+    try:
+        logger.info(f">>>list_chatbot_sources API Triggered By {vectorstore}<<<")
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/list_chatbot_sources")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        result = supabase_client.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq(
+            "chatbotName",
+            chatbot_name).execute().data
+
+        response = create_success_response(200, {"output": result})
+        logger.info(f">>>Chatbot listed successfully for {vectorstore}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in list_chatbot_sources: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/get_data_source", dependencies=[Depends(access_check_bearer)])
+async def get_data_source(vectorstore: str, source_url: str):
+    try:
+        logger.info(f">>>get_data_source API Triggered By {vectorstore}<<<")
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/get_data_source")
+        r = requests.get(source_url)
+        res = encode_to_base64(eval(r.content.decode("utf-8", errors="replace")))
+
+        response = create_success_response(200, {"output": res})
+
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in get_data_source: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/delete_chatbot_source", dependencies=[Depends(access_check_bearer)])
+async def delete_chatbot_source(request: DeleteChatbotSourceRequest):
+    vectorstore, data_source_name = request.vectorstore, request.data_source_name
+    try:
+
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/delete_chatbot_source")
+        response = supabase_client.table("ConversAI_ChatbotDataSources").delete().eq("dataSourceName",
+                                                                                     data_source_name).execute()
+        response = supabase_client.storage.from_('ConversAI').remove(f"{data_source_name}_data.json")
+
+        response = create_success_response(200, {"output": f"Successfully deleted the {data_source_name} data source."})
+
+        logger.info(f">>>Data source deleted successfully for {vectorstore}.<<<")
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in delete_chatbot_source: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/train_chatbot", dependencies=[Depends(access_check_bearer)])
+async def train_chatbot(request: TrainChatbotRequest):
+    vectorstore, url_sources = request.vectorstore, request.urls
+    logger.info(f">>>train_chatbot API Triggered By {vectorstore}<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/train_chatbot")
+        texts = []
+        sources = []
+        fileTypes = [
+            supabase_client.table("ConversAI_ChatbotDataSources").select("sourceEndpoint").eq("sourceContentURL",
+                                                                                              x).execute().data[0][
+                "sourceEndpoint"] for x in url_sources]
+        for source, fileType in zip(url_sources, fileTypes):
+            if ((fileType == "/text_pdf_extraction") | (fileType == "/image_pdf_text_extraction")):
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]
+                fileSource = file["source"]
+                texts.append(".".join(
+                    [content[key] for key in content.keys()]).replace(
+                    "\n", " "))
+                sources.append(fileSource)
+            elif fileType == "/add_text" or fileType == "/add_qa_pair":
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]["text"]
+                fileSource = file["source"]
+                texts.append(content.replace("\n", " "))
+                sources.append(fileSource)
+            elif ((fileType == "/website_url_text_extraction") | (fileType == "/youtube_transcript")):
+                r = requests.get(source)
+                file = eval(r.content.decode("utf-8", errors="replace"))
+                content = file["output"]
+                fileSource = file["source"]
+                texts.append(".".join(
+                    [content[key] for key in content.keys()]).replace(
+                    "\n", " "))
+                sources.append(fileSource)
+            else:
+                pass
+        texts = [(text, source) for text, source in zip(texts, sources)]
+        conversai_pipeline.add_document_(texts, vectorstore)
+        response = create_success_response(200, {"message": "Chatbot trained successfully."})
+        logger.info(f">>>Chatbot trained successfully for {vectorstore}.<<<")
+
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in train_chatbot: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.get("/activity_log", dependencies=[Depends(access_check_bearer)])
+async def activity_log(username: str):
+    logger.info(f">>>activityLog API Triggered By {username}<<<")
+    try:
+        response = supabase_client.table("ConversAI_ActivityLog").select("*").eq("username", username).execute().data
+
+        logger.info(f">>>Activity log fetched successfully for {username}.<<<")
+
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in activityLog: {e} for {username}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@conversai_api_router.post("/new_chatbot", dependencies=[Depends(access_check_bearer)])
+async def new_chatbot(request: NewChatbotRequest):
+    logger.info(f">>> new_chatbot API Triggered <<<")
+    try:
+        response = user_management.new_chatbot_(chatbot_name=request.chatbot_name, username=request.username)
+        logger.info(f">>> Chatbot created successfully for {request.username}.<<<")
+        return response
+
+    except Exception as e:
+        logger.error(f">>>Error in new_chatbot: {e} for {request.username}.<<<")
+        raise_http_exception(500, "Internal Server Error")

src/api/jwt_bearer.py

@@ -0,0 +1,29 @@
+"""
+Created By: ishwor subedi
+Date: 2024-09-03
+"""
+import os
+from fastapi import Depends
+from supabase import create_client
+from src import logging as logger
+from src.utils.error_handling import create_error_response, raise_http_exception
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+security = HTTPBearer()
+
+supabase_client = create_client(
+    os.getenv("SUPABASE_URL"),
+    os.getenv("SUPABASE_KEY")
+)
+
+
+async def access_check_bearer(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    access_token = credentials.credentials
+    try:
+        supabase_client.auth.get_user(access_token)
+
+    except Exception as e:
+        logger.info(f">>> Invalid access token {e}<<<")
+        raise_http_exception(code=401,
+                                   message="Invalid Access Token", details=[
+                {"info": "Invalid access token or access token expired please login again"}])

src/api/speech_api.py

@@ -0,0 +1,85 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import os
+import tempfile
+from fastapi import Form
+from fastapi import UploadFile, HTTPException, status
+from src.models.apis_models import TextToSpeechRequest
+from fastapi.routing import APIRouter
+from src.pipeline.speech_transcription_pipeline import SpeechTranscriptionPipeline
+from src import logging as logger
+from src.utils.error_handling import create_success_response, raise_http_exception
+
+speech_translator_router = APIRouter(tags=["SpeechTranscription"])
+pipeline = SpeechTranscriptionPipeline()
+
+
+@speech_translator_router.post(
+    "/text_to_speech",
+)
+async def text_to_speech(request: TextToSpeechRequest):
+    logger.info(f">>>text_to_speech API Triggered <<<")
+    try:
+        audio_bytes = pipeline.text_to_speech(request.text, request.lang, request.tld)
+        if not audio_bytes:
+            raise ValueError("Audio generation failed.")
+        response = create_success_response(code=200, data={"audio": audio_bytes})
+        logger.info(f">>>text_to_speech API success <<<")
+        return response
+    except ValueError as ve:
+        logger.info(f">>>text_to_speech API failed {ve}<<<")
+        raise_http_exception(code=400, message="Text to speech failed")
+
+    except Exception as e:
+        logger.error(f">>> Error processing text-to-speech {e}<<<")
+        raise_http_exception(code=500, message="Internal server error")
+
+
+@speech_translator_router.post(
+    "/speech_to_text",
+
+)
+async def speech_to_text(audio: UploadFile, lang: str = Form(...)):
+    logger.info(f">>>speech_to_text API Triggered <<<")
+    try:
+        audio_bytes = await audio.read()
+        if not audio_bytes:
+            logger.error(f">>> Empty audio file <<<")
+            raise ValueError("Empty audio file")
+    except Exception as e:
+        logger.error(f">>> Invalid audio file {e}<<<")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid audio file"
+        )
+
+    try:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
+            temp_audio_file.write(audio_bytes)
+            temp_audio_file_path = temp_audio_file.name
+    except Exception as e:
+        logger.error(f">>> Error creating temporary file{e} <<<")
+        raise_http_exception(code=500, message="Internal server error")
+
+    try:
+        transcript = pipeline.speech_to_text(temp_audio_file_path, lang)
+        response = create_success_response(code=200, data={"transcript": transcript})
+        logger.info(f">>>speech_to_text API success <<<")
+
+        return response
+
+    except FileNotFoundError:
+        logger.error(f">>> Temporary file not found <<<")
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Temporary file not found"
+        )
+    except Exception as e:
+        logger.error(f">>> Error processing speech-to-text {e}<<<")
+        raise_http_exception(code=500, message="Error processing speech-to-text")
+
+    finally:
+        if os.path.exists(temp_audio_file_path):
+            os.remove(temp_audio_file_path)

src/api/user_management_api.py

@@ -0,0 +1,157 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from fastapi import Depends
+from src import logging as logger
+from src.models.apis_models import *
+from fastapi.routing import APIRouter
+from src.api.jwt_bearer import access_check_bearer, supabase_client
+from src.pipeline.user_management_pipeline import SupabaseUserManagementPipeline
+from src.services.supabase.analytics.analytic_tables import track_usage
+from src.utils.error_handling import raise_http_exception, create_success_response, create_error_response
+
+user_management_api_router = APIRouter(tags=["User Management"])
+
+user_management = SupabaseUserManagementPipeline()
+
+
+@user_management_api_router.post("/user_signup")
+async def user_signup(request: UserSignupRequest):
+    logger.info(f">>>user_signup API Triggered <<<")
+    response = user_management.user_signup_(username=request.username, email=request.email, password=request.password)
+    logger.info(f">>>user_signup API Success<<<")
+
+    return response
+
+
+@user_management_api_router.post("/user_signin")
+async def user_signin(request: UserSigninRequest):
+    logger.info(f">>>user_signin API Triggered <<<")
+
+    response = user_management.user_signin_(email=request.email, password=request.password)
+    if response != None:
+        logger.info(f">>>user_signin API Success.<<<")
+        return response
+    else:
+        logger.info(f">>> Email or password is incorrect please try again.<<<")
+        response = create_error_response(400, "Email or password is incorrect please try again.")
+        return response
+
+
+@user_management_api_router.post("/get_user_data")
+async def get_user_data(request: GetUserDataRequest):
+    logger.info(f">>>get_user_data API Triggered <<<")
+    response = user_management.get_user_data_(access_token=request.access_token)
+    return response
+
+
+@user_management_api_router.post("/login_with_access_token")
+async def login_with_access_token(request: LoginWithAccessTokenRequest):
+    logger.info(f">>>login_with_access_token API Triggered <<<")
+
+    response = user_management.login_with_access_token_(access_token=request.access_token,
+                                                        refresh_token=request.refresh_token)
+    logger.info(f">>>login_with_access_token API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/set_session_data")
+async def set_session_data(request: SetSessionDataRequest):
+    logger.info(f">>> set_session_data API Triggered <<<")
+
+    response = user_management.set_session_data_(access_token=request.access_token, refresh_token=request.refresh_token,
+                                                 user_id=request.user_id)
+    return response
+
+
+@user_management_api_router.post("/sign_out")
+async def sign_out():
+    logger.info(f">>> sign_out API Triggered <<<")
+
+    response = user_management.sign_out_()
+    logger.info(f">>>sign_out API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/oauth_signin")
+async def oauth_signin():
+    logger.info(f">>> oauth_signin API Triggered <<<")
+    response = user_management.oauth_signin_()
+    logger.info(f">>>oauth_signin API Success<<<")
+    return response
+
+
+@user_management_api_router.post("/check_session")
+async def check_session():
+    logger.info(f">>>check_session API Triggered <<<")
+
+    response = user_management.check_session_()
+    return response
+
+
+@user_management_api_router.get("/get_public_chatbot")
+async def get_public_chatbots():
+    logger.info(f">>>get_public_chatbot API Triggered<<<")
+    try:
+        response = supabase_client.table("ConversAI_ChatbotInfo").select("*").eq("isPrivate", False).execute().data
+        logger.info(f">>>Public chatbots fetched successfully.<<<")
+        return response
+    except Exception as e:
+        logger.error(f">>>Error in get_public_chatbot: {e}<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@user_management_api_router.post("/public_private_check")
+async def public_or_private(request: PublicPrivateCheckRequest):
+    vectorstore, mode = request.vectorstore, request.mode
+    logger.info(f">>>public_private_check API Triggered for {vectorstore}.<<<")
+    try:
+        track_usage(supabase_client=supabase_client, vectorstore=vectorstore, endpoint="/public_private_check")
+        username, chatbot_name = vectorstore.split("$")[1], vectorstore.split("$")[2]
+        if len(mode) == 0:
+            value = (
+                supabase_client.table("ConversAI_ChatbotInfo")
+                .select("isPrivate")
+                .eq("user_id", username)
+                .eq("chatbotname", chatbot_name)
+                .execute()
+            )
+            value = value.data[0]["isPrivate"]
+            response = create_success_response(200, {"output": value})
+        else:
+            response = (
+                supabase_client.table("ConversAI_ChatbotInfo")
+                .update({"isPrivate": mode})
+                .eq("user_id", username)
+                .eq("chatbotname", chatbot_name)
+                .execute()
+            )
+            response = create_success_response(200, {"output": response})
+        logger.info(f">>>Public/Private check successful for {vectorstore}.<<<")
+        return response
+
+
+    except Exception as e:
+        logger.error(f">>>Error in public_private_check: {e} for {vectorstore}.<<<")
+        raise_http_exception(500, "Internal Server Error")
+
+
+@user_management_api_router.post("/refresh_session", dependencies=[Depends(access_check_bearer)])
+async def refresh_session(request: RefreshSessionRequest):
+    logger.info(f">>>refresh_session API Triggered <<<")
+    response = user_management.refresh_session__(refresh_token=request.refresh_token)
+    logger.info(f">>>refresh token fetched successfully.<<<")
+
+    return response
+
+
+@user_management_api_router.post("/username_creation_oauth", dependencies=[Depends(access_check_bearer)])
+async def username_creation_oauth(request: UsernameCreationOauthRequest):
+    logger.info(f">>> username_creation_oauth API Triggered <<<")
+
+    response = user_management.username_creation_oauth_(username=request.username, user_id=request.user_id,
+                                                        email=request.email)
+
+    logger.info(f">>>username creation successful.<<<")
+    return response

src/models/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/models/apis_models.py

@@ -0,0 +1,201 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from pydantic import BaseModel, EmailStr
+from typing import List, Optional
+
+
+## ---------------------------------- SpeechTranscription API Models ----------------------------------
+class TextToSpeechRequest(BaseModel):
+    text: str
+    lang: str
+    tld: str
+
+
+class SpeechToTextRequest(BaseModel):
+    lang: str
+
+
+## ---------------------------------- Chatbot API Models ----------------------------------
+
+class AddTextRequest(BaseModel):
+    vectorstore: str
+    text: str
+
+
+class AddWebsiteRequest(BaseModel):
+    website_urls: List[str]
+    vectorstore: str
+    source: str
+
+
+class AnswerQueryRequest(BaseModel):
+    query: str
+    vectorstore: str
+    llm_model: str = "llama3-70b-8192"
+
+
+class DataAnalyzerRequest(BaseModel):
+    query: str
+
+
+class GetLinksRequest(BaseModel):
+    url: str
+
+
+class YoutubeTranscriptRequest(BaseModel):
+    vectorstore: str
+    urls: List[str]
+
+
+class WebsiteUrlTextExtractionRequest(BaseModel):
+    url: str
+
+
+class WebsiteUrlTextExtractionListRequest(BaseModel):
+    urls: List[str]
+
+
+class GetCurrentCountRequest(BaseModel):
+    vectorstore: str
+
+
+class ListChatbotsRequest(BaseModel):
+    username: str
+
+
+class GetChatHistoryRequest(BaseModel):
+    vectorstore: str
+
+
+class ChatHistoryItem(BaseModel):
+    timestamp: str
+    question: str
+    response: str
+
+
+class DeleteChatbotRequest(BaseModel):
+    vectorstore: str
+
+
+class AddQAPairRequest(BaseModel):
+    vectorstore: str
+    question: str
+    answer: str
+
+
+class TrainChatbotRequest(BaseModel):
+    vectorstore: str
+    urls: list[str]
+
+
+class LoadPDFRequest(BaseModel):
+    vectorstore: str
+
+
+class LoadEditedJson(BaseModel):
+    vectorstore: str
+    data_source_name: str
+    source_endpoint: str
+    json_data: dict
+
+
+class PublicPrivateCheckRequest(BaseModel):
+    vectorstore: str
+    mode: str | None = None
+
+
+class DeleteChatbotSourceRequest(BaseModel):
+    vectorstore: str
+    data_source_name: str
+
+
+## ---------------------------------- User Management API Models ----------------------------------
+
+class UserSignupRequest(BaseModel):
+    username: str
+    email: EmailStr
+    password: str
+
+
+class UserSigninRequest(BaseModel):
+    email: EmailStr
+    password: str
+
+
+class CheckSessionRequest(BaseModel):
+    user_id: str
+
+
+class GetUserDataRequest(BaseModel):
+    access_token: str
+
+
+class RefreshSessionRequest(BaseModel):
+    refresh_token: str
+
+
+class LoginWithAccessTokenRequest(BaseModel):
+    access_token: str
+    refresh_token: str
+
+
+class UsernameCreationOauthRequest(BaseModel):
+    username: str
+    user_id: str
+    email: str
+
+
+class SetSessionDataRequest(BaseModel):
+    access_token: str
+    refresh_token: str
+    user_id: str
+
+
+class SignOutRequest(BaseModel):
+    user_id: str
+
+
+class NewChatbotRequest(BaseModel):
+    chatbot_name: str
+    username: str
+
+
+## ---------------------------------- Analytics API Models ----------------------------------
+
+
+class FeedbackRequest(BaseModel):
+    feedback: str
+    user_id: str
+    vectorstore: Optional[str]
+
+
+class UserSatisfactionRateRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class TokenUsageRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class AverageSessionInteractionRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class DailyActiveEndUserRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]
+
+
+class DailyChatCountRequest(BaseModel):
+    start_date: Optional[str]
+    end_date: Optional[str]
+    vectorstore: Optional[str]

src/models/response_handling_models.py

@@ -0,0 +1,27 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+from typing import Any, Dict
+
+from pydantic import BaseModel
+
+
+## ---------------------------------- Response Handling API Models ----------------------------------
+class SuccessResponse(BaseModel):
+    status: str
+    code: int
+    data: Dict[str, Any]
+
+
+class ErrorResponse(BaseModel):
+    status: str
+    code: int
+    error: Dict[str, Any]
+
+
+class SuccessResponseUsermanagement(BaseModel):
+    status: str
+    code: int
+    message: str
+    data: Dict[str, Any]

src/models/utls.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-29
+"""
+from pydantic import BaseModel, Field
+
+
+class FollowUps(BaseModel):
+    q1: str = Field(description="First Follow-up Question")
+    q2: str = Field(description="Second Follow-up Question")
+    q3: str = Field(description="Third Follow-up Question")

src/pipeline/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/pipeline/conversai_analytic_pipeline.py

@@ -0,0 +1,34 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-28
+"""
+import requests
+
+from src.api.conversai_api import supabase_client
+from src.services.supabase.analytics.analytic_tables import feedback_table, chat_history_table, add_feedback, \
+    track_usage
+
+
+class ConversAIAnalyticPipeline:
+    def __init__(self):
+        self.supabase_client = supabase_client
+
+    def feedback_table_(self, vectorstore):
+        return feedback_table(self.supabase_client, vectorstore)
+
+    def chat_history_table_(self, vectorstore):
+        return chat_history_table(self.supabase_client, vectorstore)
+
+    def add_feedback_(self, feedback, user_id, city, client_ip, vectorstore):
+        return add_feedback(self.supabase_client, feedback, user_id, city, client_ip, vectorstore)
+
+    def track_usage_(self, vectorstore: str, endpoint: str):
+        return track_usage(supabase_client=self.supabase_client, vectorstore=vectorstore, endpoint=endpoint)
+
+    def get_ip_info(self, ip: str):
+        try:
+            response = requests.get(f"https://ipinfo.io/{ip}/json")
+            data = response.json()
+            return data.get("city", "Unknown")
+        except Exception as e:
+            return "Unknown"

src/pipeline/conversai_pipeline.py

@@ -0,0 +1,73 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.prompts import PromptTemplate
+from src.utils.utils import load_config
+from src.services.embeddings.sentence_transformers_all_MiniLM_L6_v2_vector_embedding import \
+    all_minilm_l6_v2_vector_embedding
+from src.services.embeddings.Qdrant_BM25_embedding import qdrant_bm25_embedding
+from src.services.document.add_document import AddDocument
+from src.services.answer_query.answerquery import AnswerQuery
+from src.services.file_analyzer.data_analyzer import DataAnalyzer
+from src.services.get_links.web_scraper import WebScraper
+from src.services.ocr.replicate_ocr.replicate_ocr import ReplicateOCR as OCRService
+from src.services.pdf_extraction.image_pdf.image_pdf_text_extraction import get_text_from_image_pdf
+from src.services.pdf_extraction.text_pdf.text_pdf_extraction import extract_text_from_pdf
+from src.services.video_transcription.youtube_video_transcription.youtube_video_transcript import \
+    youtube_video_transcription
+from src.services.website_url.text_extraction_urls import extract_text_from_url_list, extract_text_from_url
+from src.utils.utils import json_parser
+
+
+class ConversAIPipeline:
+    def __init__(self):
+        prompt_template = load_config("params.yaml")["prompt"]
+        follow_up_prompt_template = load_config("params.yaml")["follow_up_prompt"]
+        prompt = ChatPromptTemplate.from_template(prompt_template)
+        json_parser_ = json_parser()
+        follow_up_prompt = PromptTemplate(
+            template=follow_up_prompt_template,
+            input_variables=["context"],
+            partial_variables={"format_instructions": json_parser_.get_format_instructions()},
+        )
+        self.vector_embedding = all_minilm_l6_v2_vector_embedding()
+        self.sparse_embedding = qdrant_bm25_embedding()
+        self.add_document_service = AddDocument(self.vector_embedding, self.sparse_embedding)
+        self.answer_query_service = AnswerQuery(vector_embedding=self.vector_embedding,
+                                                sparse_embedding=self.sparse_embedding, prompt=prompt,
+                                                follow_up_prompt=follow_up_prompt, json_parser=json_parser_)
+        self.data_analyzer = DataAnalyzer()
+        self.get_website_links = WebScraper()
+        self.ocr_service = OCRService()
+
+    def add_document_(self, texts: list[tuple[str]], vectorstore: str):
+        return self.add_document_service.add_documents(texts=texts, vectorstore=vectorstore)
+
+    def answer_query_(self, query: str, vectorstore: str, llm_model: str = "llama-3.1-70b-versatile"):
+        output, follow_up_questions, source = self.answer_query_service.answer_query(query=query,
+                                                                                     vectorstore=vectorstore,
+                                                                                     llmModel=llm_model)
+        return output, follow_up_questions, source
+
+    def data_analyzer_(self, query: str, dataframe):
+        return self.data_analyzer.analyze_data(query=query, dataframe=dataframe)
+
+    def get_links_(self, url: str, timeout: int):
+        return self.get_website_links.get_links(url=url, timeout=timeout)
+
+    def image_pdf_text_extraction_(self, image_pdf: bytes):
+        return get_text_from_image_pdf(pdf_bytes=image_pdf)
+
+    def text_pdf_extraction_(self, pdf: str):
+        return extract_text_from_pdf(pdf_path=pdf)
+
+    def youtube_transcript_(self, url: list):
+        return youtube_video_transcription(youtube_video_url=url)
+
+    def website_url_text_extraction_(self, url: str):
+        return extract_text_from_url(url=url)
+
+    def website_url_text_extraction_list_(self, urls: list):
+        return extract_text_from_url_list(urls=urls)

src/pipeline/speech_transcription_pipeline.py

@@ -0,0 +1,20 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from src.services.speech_to_text.speech_to_text_replicate import SpeechToTextReplicate
+from src.services.text_to_speech.text_to_speech_gtts import TextToSpeech
+
+
+class SpeechTranscriptionPipeline:
+    def __init__(self):
+        self.speech_to_text_ = SpeechToTextReplicate()
+        self.text_to_speech_ = TextToSpeech()
+
+    def text_to_speech(self, text: str, lang: str, tld: str) -> str:
+        speech = self.text_to_speech_.conversion(text, lang, tld)
+        return speech
+
+    def speech_to_text(self, audio, lang: str) -> str:
+        transcript_with_timestamp, transcript = self.speech_to_text_.transcribe_audio(audio=audio, language=lang)
+        return transcript

src/pipeline/user_management_pipeline.py

@@ -0,0 +1,76 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from src.services.supabase.user_management.user_service import UserManagement
+from src.services.supabase.user_management.chatbot_management import SupabaseChatoBotManagement
+from src.services.supabase.conversai_setup.conversai_user_db_setup import ConversAIUserDBSetup
+from src.services.supabase.user_management.chat_history import get_chat_history
+from supabase.client import create_client
+from qdrant_client import QdrantClient
+import os
+
+SUPABASE_URL = os.getenv("SUPABASE_URL")
+SUPABASE_KEY = os.getenv("SUPABASE_KEY")
+
+QDRANT_URL = os.getenv("QDRANT_URL")
+QDRANT_API_KEY = os.getenv("QDRANT_API_KEY")
+
+
+class SupabaseUserManagementPipeline:
+    def __init__(self):
+        self.supabase_client = create_client(SUPABASE_URL, SUPABASE_KEY)
+        self.qdrant_client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
+        self.user_management = UserManagement(supabase_client=self.supabase_client)
+        self.chatbot_management = SupabaseChatoBotManagement(supabase_client=self.supabase_client,
+                                                             qdrant_client=self.qdrant_client)
+        self.user_db_setup = ConversAIUserDBSetup(supabase_client=self.supabase_client)
+
+    def user_signup_(self, username: str, email: str, password: str) -> dict:
+        return self.user_management.user_signup(username=username, email=email, password=password)
+
+    def user_signin_(self, email: str, password: str) -> dict:
+        return self.user_management.user_signin(email=email, password=password)
+
+    def check_session_(self) -> dict:
+        return self.user_management.check_session()
+
+    def get_user_data_(self, access_token: str) -> dict:
+        return self.user_management.get_user_data(access_token=access_token)
+
+    def refresh_session__(self, refresh_token: str) -> dict:
+        return self.user_management.refresh_session_(refresh_token=refresh_token)
+
+    def login_with_access_token_(self, access_token: str, refresh_token: str) -> dict:
+        return self.user_management.login_with_access_token(access_token=access_token, refresh_token=refresh_token)
+
+    def username_creation_oauth_(self, username: str, user_id: str, email: str):
+        return self.user_management.user_name_creation_oauth(user_id=user_id, username=username, email=email)
+
+    def set_session_data_(self, access_token: str, refresh_token: str, user_id: str):
+        return self.user_management.set_session_data(access_token=access_token, refresh_token=refresh_token,
+                                                     user_id=user_id)
+
+    def sign_out_(self):
+        return self.user_management.sign_out_()
+
+    def oauth_signin_(self) -> dict:
+        return self.user_management.oauth()
+
+    def new_chatbot_(self, chatbot_name: str, username: str):
+        return self.chatbot_management.new_chatbot(chatbot_name=chatbot_name, username=username)
+
+    def get_chat_history_(self, vectorstore: str):
+        return get_chat_history(vectorstore=vectorstore, supabase_client=self.supabase_client)
+
+    def delete_table(self, table_name: str):
+        return self.chatbot_management.delete_table(table_name=table_name)
+
+    def list_tables(self, username: str):
+        return self.chatbot_management.list_tables(username=username)
+
+    def create_data_source_name(self, source_name: str, username: str):
+        return self.chatbot_management.create_data_source_name(source_name=source_name, username=username)
+
+    def delete_qdrant_cluster(self, vectorstorename):
+        self.qdrant_client.delete_collection(collection_name=vectorstorename)

src/services/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/services/answer_query/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/answer_query/answerquery.py

@@ -0,0 +1,93 @@
+from langchain_groq import ChatGroq
+from langchain_core.runnables import RunnablePassthrough, RunnableLambda
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables.history import RunnableWithMessageHistory 
+from langchain.memory import ChatMessageHistory
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.document_compressors import FlashrankRerank 
+from langchain_core.chat_history import BaseChatMessageHistory
+from src.services.vector_db.qdrent.upload_document import upload_document_existing_collection, \
+    answer_query_from_existing_collection
+
+
+class AnswerQuery:
+    def __init__(self, prompt, vector_embedding, sparse_embedding, follow_up_prompt, json_parser):
+        self.chat_history_store = {}
+        self.compressor = FlashrankRerank()
+        self.vector_embed = vector_embedding
+        self.sparse_embed = sparse_embedding
+        self.prompt = prompt
+        self.follow_up_prompt = follow_up_prompt
+        self.json_parser = json_parser        
+
+    def format_docs(self, docs: str):
+        global sources
+        global temp_context
+        sources = []
+        context = ""
+        for doc in docs:
+            context += f"{doc.page_content}\n\n\n"
+            source = doc.metadata
+            source = source["source"]
+            sources.append(source)
+        if context == "":
+            context = "No context found"
+        else:
+            pass
+        sources = list(set(sources))
+        temp_context = context
+        return context
+
+    def answer_query(self, query: str, vectorstore: str, llmModel: str = "llama-3.1-70b-versatile"):
+        global sources
+        global temp_context
+        vector_store_name = vectorstore
+        vector_store = answer_query_from_existing_collection(vector_embed=self.vector_embed,
+                                                             sparse_embed=self.sparse_embed,
+                                                             vectorstore=vectorstore)
+
+        retriever = vector_store.as_retriever(search_type="mmr", search_kwargs={"k": 10, "fetch_k": 20})
+        compression_retriever = ContextualCompressionRetriever(
+            base_compressor = self.compressor, base_retriever = retriever
+        )
+        brain_chain = (
+                {"context": RunnableLambda(lambda x: x["question"]) | compression_retriever | RunnableLambda(self.format_docs),
+                 "question": RunnableLambda(lambda x: x["question"]),
+                 "chatHistory": RunnableLambda(lambda x: x["chatHistory"])}
+                | self.prompt
+                | ChatGroq(model=llmModel, temperature=0.75, max_tokens=512)
+                | StrOutputParser()
+        )
+        message_chain = RunnableWithMessageHistory(
+            brain_chain,
+            self.get_session_history,
+            input_messages_key="question",
+            history_messages_key="chatHistory"
+        )
+        chain = RunnablePassthrough.assign(messages_trimmed=self.trim_messages) | message_chain
+        follow_up_chain = self.follow_up_prompt | ChatGroq(model_name="llama-3.1-70b-versatile",
+                                                           temperature=0) | self.json_parser
+
+        output = chain.invoke(
+            {"question": query},
+            {"configurable": {"session_id": vector_store_name}}
+        )
+        follow_up_questions = follow_up_chain.invoke({"context": temp_context})
+
+        return output, follow_up_questions, sources
+
+    def trim_messages(self, chain_input):
+        for store_name in self.chat_history_store:
+            messages = self.chat_history_store[store_name].messages
+            if len(messages) <= 1:
+                pass
+            else:
+                self.chat_history_store[store_name].clear()
+                for message in messages[-1:]:
+                    self.chat_history_store[store_name].add_message(message)
+        return True
+
+    def get_session_history(self, session_id: str) -> BaseChatMessageHistory:
+        if session_id not in self.chat_history_store:
+            self.chat_history_store[session_id] = ChatMessageHistory()
+        return self.chat_history_store[session_id]

src/services/document/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/document/add_document.py

@@ -0,0 +1,30 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import string
+from uuid import uuid4
+from langchain.docstore.document import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from src.services.vector_db.qdrent.upload_document import upload_document_existing_collection
+
+
+class AddDocument:
+    def __init__(self, vector_embedding, sparse_embedding):
+        self.vector_embed = vector_embedding
+        self.sparse_embed = sparse_embedding
+
+    def add_documents(self, texts: list[tuple[str]], vectorstore: str):
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1500,
+            chunk_overlap=250,
+            add_start_index=True
+        )
+        sources = [textTuple[1] for textTuple in texts]
+        texts = [textTuple[0].replace("\n", " ") for textTuple in texts]
+        texts = [text.translate(str.maketrans('', '', string.punctuation.replace(".", ""))) for text in texts]
+        texts = [Document(page_content=text, metadata={"source": source}) for text, source in zip(texts, sources)]
+        documents = splitter.split_documents(texts)
+        upload_document_existing_collection(vector_embed=self.vector_embed,
+                                            sparse_embed=self.sparse_embed,
+                                            vectorstore=vectorstore, documents=documents)

src/services/embeddings/BGE-M3_vector_embedding.py

@@ -0,0 +1,13 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_huggingface import HuggingFaceEmbeddings
+
+
+def bge_m3_vector_embedding():
+    return HuggingFaceEmbeddings(
+        model_name="BAAI/bge-m3",
+        model_kwargs={"device": "cuda"},
+        encode_kwargs={"normalize_embeddings": True}
+    )

src/services/embeddings/Qdrant_BM25_embedding.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_qdrant import FastEmbedSparse
+
+
+def qdrant_bm25_embedding():
+    instance = FastEmbedSparse(model="Qdrant/BM25", threads=20, parallel=0)
+
+    return instance

src/services/embeddings/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/embeddings/sentence_transformers_all_MiniLM_L6_v2_vector_embedding.py

@@ -0,0 +1,20 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+from langchain_huggingface import HuggingFaceEmbeddings
+from src.utils.utils import load_ini_config
+
+
+def all_minilm_l6_v2_vector_embedding():
+    config = load_ini_config("config.ini")
+
+    model_kwargs = {"device": config.get('all_mini_l6_v2_vector_embedding', 'device')}
+    encode_kwargs = {"normalize_embeddings": config.get('all_mini_l6_v2_vector_embedding', 'normalize_embeddings')}
+
+    all_minilm_object = HuggingFaceEmbeddings(
+        model_name="BAAI/bge-base-en-v1.5",
+        model_kwargs=model_kwargs,
+        encode_kwargs=encode_kwargs
+    )
+    return all_minilm_object

src/services/file_analyzer/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/file_analyzer/data_analyzer.py

@@ -0,0 +1,30 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import os
+import base64
+from langchain_groq import ChatGroq
+from pandasai import SmartDataframe
+from src.utils.utils import load_ini_config
+
+
+class DataAnalyzer:
+    def __init__(self):
+        self.config = load_ini_config("config.ini")
+
+        self.llm_config = ChatGroq(name=self.config.get('data_analyzer', 'groq_llm_name'))
+        self.additional_query = self.config.get('data_analyzer', 'additional_query')
+
+    def analyze_data(self, query, dataframe):
+        query += self.additional_query
+        df = SmartDataframe(dataframe,
+                            config={"llm": self.llm_config,
+                                    "verbose": bool(self.config.get("data_analyzer", "verbose"))})
+        response = df.chat(query)
+        if os.path.isfile(response):
+            with open(response, "rb") as file:
+                b64string = base64.b64encode(file.read()).decode("utf-8")
+            return f"data:image/png;base64,{b64string}"
+        else:
+            return response

src/services/get_links/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/get_links/web_scraper.py

@@ -0,0 +1,41 @@
+import time
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin
+
+
+class WebScraper:
+    def __init__(self):
+        pass
+
+    def get_links(self,url: str, timeout=30):
+        start = time.time()
+
+        def get_links_from_page(url: str) -> list:
+            response = requests.get(url)
+            soup = BeautifulSoup(response.content, "lxml")
+            anchors = soup.find_all("a")
+            links = []
+            for anchor in anchors:
+                if "href" in anchor.attrs:
+                    if urlparse(anchor.attrs["href"]).netloc == urlparse(url).netloc:
+                        links.append(anchor.attrs["href"])
+                    elif not anchor.attrs["href"].startswith(("//", "file", "javascript", "tel", "mailto", "http")):
+                        links.append(urljoin(url + "/", anchor.attrs["href"]))
+                    else:
+                        pass
+                    links = [link for link in links if "#" not in link]
+                    links = list(set(links))
+                else:
+                    continue
+            return links
+
+        links = get_links_from_page(url)
+        unique_links = set()
+        for link in links:
+            now = time.time()
+            if now - start > timeout:
+                break
+            else:
+                unique_links = unique_links.union(set(get_links_from_page(link)))
+        return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in unique_links]))

src/services/ocr/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/ocr/easy_ocr/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/ocr/easy_ocr/easy_ocr_.py

@@ -0,0 +1,19 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import cv2
+import easyocr
+import numpy as np
+from src.utils.utils import load_ini_config
+
+
+class EasyOCR_:
+    def __init__(self):
+        self.config = load_ini_config("config.ini")
+        self.reader = easyocr.Reader([self.config.get('easy_ocr', 'language')],
+                                     gpu=bool(self.config.get('easy_ocr', 'gpu')),
+                                     model_storage_directory=self.config.get('easy_ocr', 'model_path'))
+
+    def read_text(self, image: np.array):
+        return self.reader.readtext(image, paragraph=True)

src/services/ocr/replicate_ocr/__init__.py

@@ -0,0 +1,5 @@
+"""
+project @ convers-ai
+created @ 2024-11-06
+author  @ github.com/ishworrsubedii
+"""

src/services/ocr/replicate_ocr/replicate_ocr.py

@@ -0,0 +1,35 @@
+"""
+project @ convers-ai
+created @ 2024-11-06
+author  @ github.com/ishworrsubedii
+"""
+import base64
+import numpy as np
+import replicate
+
+
+def replicate_run(input):
+    output = replicate.run(
+        "abiruyt/text-extract-ocr:a524caeaa23495bc9edc805ab08ab5fe943afd3febed884a4f3747aa32e9cd61",
+        input=input
+    )
+    return output
+
+
+class ReplicateOCR:
+    def __init__(self):
+        pass
+
+    def read_text(self, image_path: str):
+        with open(image_path, "rb") as image_file:
+            base_64_image = base64.b64encode(image_file.read()).decode('utf-8')
+
+        input_image_uri= f"data:image/png;base64,{base_64_image}"
+        input = {
+            "image": input_image_uri,
+        }
+        output = replicate_run(input)
+        output = output.replace("\n", " ").replace("\t", " ")
+
+
+        return output

src/services/pdf_extraction/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/pdf_extraction/image_pdf/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/pdf_extraction/image_pdf/image_pdf_text_extraction.py

@@ -0,0 +1,29 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import base64
+
+import numpy as np
+from pdf2image import convert_from_bytes
+from src.services.ocr.replicate_ocr.replicate_ocr import ReplicateOCR
+
+easy_ocr = ReplicateOCR()
+
+
+def get_text_from_image_pdf(pdf_bytes: bytes):
+    def getText(image):
+        global reader
+
+        return "\n".join([text[1] for text in easy_ocr.read_text(np.array(image))])
+
+    all_images = convert_from_bytes(pdf_bytes)
+    texts = [getText(image) for image in all_images]
+    return {x + 1: y for x, y in enumerate(texts)}
+
+
+if __name__ == '__main__':
+    pdf = "/home/ishwor/Downloads/Invoice-cup.pdf"
+    with open(pdf, "rb") as f:
+        p = f.read()
+    get_text_from_image_pdf(pdf_bytes=p)

src/services/pdf_extraction/text_pdf/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""

src/services/pdf_extraction/text_pdf/text_pdf_extraction.py

@@ -0,0 +1,21 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""
+import base64
+import pymupdf
+from concurrent.futures import ThreadPoolExecutor
+
+
+def extract_text_from_page(page):
+    text = page.get_text()
+    return text
+
+
+def extract_text_from_pdf(pdf_path: str):
+    doc = pymupdf.open(pdf_path)
+    pages = [doc.load_page(i) for i in range(len(doc))]
+    with ThreadPoolExecutor() as executor:
+        texts = list(executor.map(extract_text_from_page, pages))
+    doc.close()
+    return {x + 1: y for x, y in enumerate(texts)}

src/services/speech_to_text/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-23
+"""