first commit

.chainlit/config.toml

@@ -0,0 +1,121 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Authorized origins
+allow_origins = ["*"]
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Show the prompt playground
+prompt_playground = true
+
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Automatically tag threads with the current chat profile (if a chat profile is used)
+auto_tag_thread = true
+
+# Authorize users to spontaneously upload files with messages
+[features.spontaneous_file_upload]
+    enabled = true
+    accept = ["*/*"]
+    max_files = 20
+    max_size_mb = 500
+
+[features.audio]
+    # Threshold for audio recording
+    min_decibels = -45
+    # Delay for the user to start speaking in MS
+    initial_silence_timeout = 3000
+    # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop.
+    silence_timeout = 1500
+    # Above this duration (MS), the recording will forcefully stop.
+    max_duration = 15000
+    # Duration of the audio chunks in MS
+    chunk_duration = 1000
+    # Sample rate of the audio
+    sample_rate = 44100
+
+[UI]
+# Name of the app and chatbot.
+name = "Chatbot"
+
+# Show the readme while the thread is empty.
+show_readme_as_default = true
+
+# Description of the app and chatbot. This is used for HTML tags.
+# description = ""
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# The default value for the expand messages settings.
+default_expand_messages = false
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Specify a Javascript file that can be used to customize the user interface.
+# The Javascript file can be served from the public directory.
+# custom_js = "/public/test.js"
+
+# Specify a custom font url.
+# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap"
+
+# Specify a custom meta image url.
+# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
+
+# Specify a custom build directory for the frontend.
+# This can be used to customize the frontend code.
+# Be careful: If this is a relative path, it should not start with a slash.
+# custom_build = "./public/build"
+
+[UI.theme]
+    #layout = "wide"
+    #font_family = "Inter, sans-serif"
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.dark.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+
+[meta]
+generated_by = "1.1.202"

.chainlit/translations/en-US.json

@@ -0,0 +1,231 @@
+{
+    "components": {
+        "atoms": {
+            "buttons": {
+                "userButton": {
+                    "menu": {
+                        "settings": "Settings",
+                        "settingsKey": "S",
+                        "APIKeys": "API Keys",
+                        "logout": "Logout"
+                    }
+                }
+            }
+        },
+        "molecules": {
+            "newChatButton": {
+                "newChat": "New Chat"
+            },
+            "tasklist": {
+                "TaskList": {
+                    "title": "\ud83d\uddd2\ufe0f Task List",
+                    "loading": "Loading...",
+                    "error": "An error occured"
+                }
+            },
+            "attachments": {
+                "cancelUpload": "Cancel upload",
+                "removeAttachment": "Remove attachment"
+            },
+            "newChatDialog": {
+                "createNewChat": "Create new chat?",
+                "clearChat": "This will clear the current messages and start a new chat.",
+                "cancel": "Cancel",
+                "confirm": "Confirm"
+            },
+            "settingsModal": {
+                "settings": "Settings",
+                "expandMessages": "Expand Messages",
+                "hideChainOfThought": "Hide Chain of Thought",
+                "darkMode": "Dark Mode"
+            },
+            "detailsButton": {
+                "using": "Using",
+                "running": "Running",
+                "took_one": "Took {{count}} step",
+                "took_other": "Took {{count}} steps"
+            },
+            "auth": {
+                "authLogin": {
+                    "title": "Login to access the app.",
+                    "form": {
+                        "email": "Email address",
+                        "password": "Password",
+                        "noAccount": "Don't have an account?",
+                        "alreadyHaveAccount": "Already have an account?",
+                        "signup": "Sign Up",
+                        "signin": "Sign In",
+                        "or": "OR",
+                        "continue": "Continue",
+                        "forgotPassword": "Forgot password?",
+                        "passwordMustContain": "Your password must contain:",
+                        "emailRequired": "email is a required field",
+                        "passwordRequired": "password is a required field"
+                    },
+                    "error": {
+                        "default": "Unable to sign in.",
+                        "signin": "Try signing in with a different account.",
+                        "oauthsignin": "Try signing in with a different account.",
+                        "redirect_uri_mismatch": "The redirect URI is not matching the oauth app configuration.",
+                        "oauthcallbackerror": "Try signing in with a different account.",
+                        "oauthcreateaccount": "Try signing in with a different account.",
+                        "emailcreateaccount": "Try signing in with a different account.",
+                        "callback": "Try signing in with a different account.",
+                        "oauthaccountnotlinked": "To confirm your identity, sign in with the same account you used originally.",
+                        "emailsignin": "The e-mail could not be sent.",
+                        "emailverify": "Please verify your email, a new email has been sent.",
+                        "credentialssignin": "Sign in failed. Check the details you provided are correct.",
+                        "sessionrequired": "Please sign in to access this page."
+                    }
+                },
+                "authVerifyEmail": {
+                    "almostThere": "You're almost there! We've sent an email to ",
+                    "verifyEmailLink": "Please click on the link in that email to complete your signup.",
+                    "didNotReceive": "Can't find the email?",
+                    "resendEmail": "Resend email",
+                    "goBack": "Go Back",
+                    "emailSent": "Email sent successfully.",
+                    "verifyEmail": "Verify your email address"
+                },
+                "providerButton": {
+                    "continue": "Continue with {{provider}}",
+                    "signup": "Sign up with {{provider}}"
+                },
+                "authResetPassword": {
+                    "newPasswordRequired": "New password is a required field",
+                    "passwordsMustMatch": "Passwords must match",
+                    "confirmPasswordRequired": "Confirm password is a required field",
+                    "newPassword": "New password",
+                    "confirmPassword": "Confirm password",
+                    "resetPassword": "Reset Password"
+                },
+                "authForgotPassword": {
+                    "email": "Email address",
+                    "emailRequired": "email is a required field",
+                    "emailSent": "Please check the email address {{email}} for instructions to reset your password.",
+                    "enterEmail": "Enter your email address and we will send you instructions to reset your password.",
+                    "resendEmail": "Resend email",
+                    "continue": "Continue",
+                    "goBack": "Go Back"
+                }
+            }
+        },
+        "organisms": {
+            "chat": {
+                "history": {
+                    "index": {
+                        "showHistory": "Show history",
+                        "lastInputs": "Last Inputs",
+                        "noInputs": "Such empty...",
+                        "loading": "Loading..."
+                    }
+                },
+                "inputBox": {
+                    "input": {
+                        "placeholder": "Type your message here..."
+                    },
+                    "speechButton": {
+                        "start": "Start recording",
+                        "stop": "Stop recording"
+                    },
+                    "SubmitButton": {
+                        "sendMessage": "Send message",
+                        "stopTask": "Stop Task"
+                    },
+                    "UploadButton": {
+                        "attachFiles": "Attach files"
+                    },
+                    "waterMark": {
+                        "text": "Built with"
+                    }
+                },
+                "Messages": {
+                    "index": {
+                        "running": "Running",
+                        "executedSuccessfully": "executed successfully",
+                        "failed": "failed",
+                        "feedbackUpdated": "Feedback updated",
+                        "updating": "Updating"
+                    }
+                },
+                "dropScreen": {
+                    "dropYourFilesHere": "Drop your files here"
+                },
+                "index": {
+                    "failedToUpload": "Failed to upload",
+                    "cancelledUploadOf": "Cancelled upload of",
+                    "couldNotReachServer": "Could not reach the server",
+                    "continuingChat": "Continuing previous chat"
+                },
+                "settings": {
+                    "settingsPanel": "Settings panel",
+                    "reset": "Reset",
+                    "cancel": "Cancel",
+                    "confirm": "Confirm"
+                }
+            },
+            "threadHistory": {
+                "sidebar": {
+                    "filters": {
+                        "FeedbackSelect": {
+                            "feedbackAll": "Feedback: All",
+                            "feedbackPositive": "Feedback: Positive",
+                            "feedbackNegative": "Feedback: Negative"
+                        },
+                        "SearchBar": {
+                            "search": "Search"
+                        }
+                    },
+                    "DeleteThreadButton": {
+                        "confirmMessage": "This will delete the thread as well as it's messages and elements.",
+                        "cancel": "Cancel",
+                        "confirm": "Confirm",
+                        "deletingChat": "Deleting chat",
+                        "chatDeleted": "Chat deleted"
+                    },
+                    "index": {
+                        "pastChats": "Past Chats"
+                    },
+                    "ThreadList": {
+                        "empty": "Empty...",
+                        "today": "Today",
+                        "yesterday": "Yesterday",
+                        "previous7days": "Previous 7 days",
+                        "previous30days": "Previous 30 days"
+                    },
+                    "TriggerButton": {
+                        "closeSidebar": "Close sidebar",
+                        "openSidebar": "Open sidebar"
+                    }
+                },
+                "Thread": {
+                    "backToChat": "Go back to chat",
+                    "chatCreatedOn": "This chat was created on"
+                }
+            },
+            "header": {
+                "chat": "Chat",
+                "readme": "Readme"
+            }
+        }
+    },
+    "hooks": {
+        "useLLMProviders": {
+            "failedToFetchProviders": "Failed to fetch providers:"
+        }
+    },
+    "pages": {
+        "Design": {},
+        "Env": {
+            "savedSuccessfully": "Saved successfully",
+            "requiredApiKeys": "Required API Keys",
+            "requiredApiKeysInfo": "To use this app, the following API keys are required. The keys are stored on your device's local storage."
+        },
+        "Page": {
+            "notPartOfProject": "You are not part of this project."
+        },
+        "ResumeButton": {
+            "resumeChat": "Resume Chat"
+        }
+    }
+}

.gitignore

@@ -0,0 +1,112 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environment
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.venv/
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+pytest-debug.log
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/_site/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+.env.*
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VS Code
+.vscode/
+
+# IntelliJ
+.idea/
+
+# Mac OS
+.DS_Store

TODO.md

@@ -0,0 +1,78 @@
+## Initial System Efficiency Strategy
+1. Query generator system
+    a. Discovery
+    b. Tokens
+2. Prompt enhancer system
+3. Typesense instance for (short cached)
+    a. user actions similarity and recomendation/suggestion
+    b. function path and execution graph search
+    c. external data fetches and proactive data fetch
+4. Function path and execution graph generator class
+5. Timescale instance for  metric measurements and ranking
+    a. Function execution time
+    b. Gas spend
+    c. Integration latency
+    d. Prompt -> answer latency
+6. Proactive cache enrichment based on user activity
+7. Proactive knowledge graph update based on market activity and user interests
+    a. KG pipeline flow
+    b. Progressive cache and hot context for llm as flow progresses
+8. Purchase suggestion engines for users
+9. Subtle user education to increase chances of purchases
+10. Interest provoking alerts
+    a. Toasts
+    b. Email
+    c. Highlights
+11. Short form content generation engine
+    a. Giphs
+    b. Memes
+    c. Short video highlights
+    d. Short voice highlights in background as user interacts with system
+    e. Short top gainer and looser animations
+12. User PnL stats animations layerd on system prediction performance
+13. User pseudo trading system gamified
+14. Fast and easy token purchase for boosted assets (LP partners)
+15. Index system for users (create, manage and adjust) for short term and long term
+16. User budgeting and financial planning system with scores
+17. Users global engagment score and leaaderboard system with point awards
+18. Fundamentaal analysis feature
+19. Signal and listing alerts
+20. Copy trading service
+21. Bot trading service
+22. Persive-Agressive investor potfolio execution and management service
+23. Stock trading service
+24. Utility payments
+25. Travel service
+    a. Cheap flight paths
+    b. Esim
+    c. Travel expense on crypto
+    d. Airtime and data
+    e. Vouchers and points
+
+
+## Infra
+1. Async everthing
+2. CI/CD pipeline
+3. Strict typed input and output validations
+4. Debugging logs
+5. Proper general logging and export to log service
+6. LLM load balancing and redundancy
+7. LLM caching
+8. Local and secure micro LLMs servers (Fly or RunPod)
+9. Proper backups of all infra resources
+10. Cloudflare captcha and WAF
+11. User traffic monitoring and active banning system
+12. Ofac and sanctions checks
+13. KYC flows
+14. KYC check service offering to 3rd party
+15. V1 SDK realease
+16. V1 developer contribution campaings
+
+
+## Priority
+1. Host TypeSence or Upstash Redis, Timescale DB, LiteLLM, Ollama, FalkorDB, 
+    a. compare Fly and Digital Ocean
+2. Create secret store and integrate
+3. Create ChatXBT host scalabe hosting (fly or Digital Ocean or RunPod)
+4. Set alerts for billing and uptime
+5. Set error alerts and llm investigation

ai_functions.py

@@ -0,0 +1,60 @@
+from pydantic import BaseModel, Field
+from typing import Dict, Optional, Type
+
+from data_sources.coin_gecko import CoinGecko
+from src.data_sources.cryptocompare import CryptoCompare
+
+from langchain.tools.base import BaseTool
+
+class CryptoCoinPrice(BaseModel):
+    """Represents the prices of a coin in various currencies."""
+    prices: Dict[str, float] = Field(..., description="Prices in various currencies")
+
+class CryptoCoinPriceData(BaseModel):
+    """Encapsulates both CoinGecko and CryptoCompare price data."""
+    coingecko_price: Dict[str, CryptoCoinPrice] = Field(..., description="CoinGecko prices for various coins")
+    crypto_compare_price: Dict[str, CryptoCoinPrice] = Field(..., description="CryptoCompare prices for various coins")
+
+class PriceInput(BaseModel):
+    coin_id: str = Field(..., description="The ID of the cryptocurrency coin to retrieve prices for")
+    vs_currency: str = Field("usd", description="The currency to compare against")
+
+class CryptoCoinPriceOutput(BaseModel):
+    price_data: CryptoCoinPriceData
+
+class CryptoCoinPriceTool(BaseTool):
+    name = "CryptoCoinPriceTool"
+    description = "Fetches price data for a given cryptocurrency coin from CoinGecko and CryptoCompare"
+    args_schema: Type[BaseModel] = PriceInput
+    return_direct: bool = True
+    
+    def __init__(self, id: Optional[str] = None):
+        self.id = id
+        self.coingecko = CoinGecko()
+        self.crypto_compare = CryptoCompare()
+
+    def _run(self, coin_id: str, vs_currency: str = "usd") -> CryptoCoinPriceData:
+        coingecko_price_data = self.coingecko.get_coin_price(ids=[coin_id], vs_currencies=[vs_currency])
+        crypto_compare_price_data = self.crypto_compare.get_coin_price(ids=[coin_id], vs_currencies=[vs_currency])
+
+        coingecko_price = {}
+        crypto_compare_price = {}
+
+        if coin_id in coingecko_price_data:
+            coingecko_price[coin_id] = CryptoCoinPrice(prices=coingecko_price_data[coin_id])
+        else:
+            print(f"Warning: CoinGecko data for {coin_id} not found.")
+
+        if coin_id.upper() in crypto_compare_price_data:
+            crypto_compare_price[coin_id] = CryptoCoinPrice(prices=crypto_compare_price_data[coin_id.upper()])
+        else:
+            print(f"Warning: CryptoCompare data for {coin_id} not found.")
+
+        return CryptoCoinPriceData(
+            coingecko_price=coingecko_price,
+            crypto_compare_price=crypto_compare_price
+        )
+
+    def __call__(self, inputs: PriceInput) -> CryptoCoinPriceOutput:
+        price_data = self._run(inputs.coin_id, inputs.vs_currency)
+        return CryptoCoinPriceOutput(price_data=price_data)

chat-app.py

@@ -0,0 +1,55 @@
+from dotenv import load_dotenv
+
+from langchain.chains import LLMMathChain
+from langchain.llms.openai import OpenAI
+from langchain.chat_models import ChatOpenAI
+from langchain.utilities.serpapi import SerpAPIWrapper
+from langchain.agents import initialize_agent, Tool, AgentExecutor
+import chainlit as cl
+
+from src.tools.crypto_coin_price_tool import CryptoCoinPriceTool
+
+load_dotenv()
+
+@cl.on_chat_start
+def start():
+    llm = ChatOpenAI(temperature=0, streaming=True)
+    llm1 = OpenAI(temperature=0, streaming=True)
+    search = SerpAPIWrapper()
+    get_crypto_coin_price = CryptoCoinPriceTool()
+    llm_math_chain = LLMMathChain.from_llm(llm=llm, verbose=True)
+
+    tools = [
+        Tool(
+            name="Search",
+            func=search.run,
+            description="useful for when you need to answer questions about current events. You should ask targeted questions",
+            handle_tool_error=True,
+        ),
+        Tool(
+            name="Calculator",
+            func=llm_math_chain.run,
+            description="useful for when you need to answer questions about math",
+            handle_tool_error=True,
+        ),
+        Tool(
+            name=get_crypto_coin_price.name,
+            func=get_crypto_coin_price.run,
+            description=get_crypto_coin_price.description,
+            handle_tool_error=True,
+        ),
+    ]
+    agent = initialize_agent(
+        tools, llm1, agent="chat-zero-shot-react-description", verbose=True, handle_parsing_errors=True
+    )
+    cl.user_session.set("agent", agent)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    agent = cl.user_session.get("agent")  # type: AgentExecutor
+    cb = cl.LangchainCallbackHandler(stream_final_answer=True)
+
+    await cl.make_async(agent.run)(message.content, callbacks=[cb])
+
+    

lite-llm-config.yaml

@@ -0,0 +1,15 @@
+model_list: 
+  - model_name: gpt-3.5-turbo # user-facing model alias
+    litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
+      model: azure/<your-deployment-name>
+      api_base: <your-azure-api-endpoint>
+      api_key: <your-azure-api-key>
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: azure/gpt-turbo-small-ca
+      api_base: https://my-endpoint-canada-berri992.openai.azure.com/
+      api_key: <your-azure-api-key>
+  - model_name: vllm-model
+    litellm_params:
+      model: openai/<your-model-name>
+      api_base: <your-api-base> # e.g. http://0.0.0.0:3000

reference.md

@@ -0,0 +1,23 @@
+## py-jsonq
+https://github.com/s1s1ty/py-jsonq
+py-jsonq is a simple, elegant Python package to Query over any type of JSON Data. It'll make your life easier by giving the flavour of an ORM-like query on your JSON.
+
+## JMESPath Python: JSON Query Language
+https://python.land/data-processing/working-with-json/jmespath
+
+## fastapi-cloudevents
+Allows to easily consume and produce CloudEvents over REST API.
+https://pypi.org/project/fastapi-cloudevents/
+
+## Pydantic
+1. Logfire
+    https://pydantic.dev/logfire
+    https://pydantic.dev/opensource
+    https://docs.pydantic.dev/logfire
+2. Settings & Config - https://docs.pydantic.dev/latest/concepts/pydantic_settings/
+
+## Dev Tools
+3. https://docs.astral.sh/ruff/
+4. https://tox.wiki/en/4.15.1/index.html
+5. https://black.readthedocs.io/en/stable/
+6. https://pycqa.github.io/isort/

requirements.bk.txt

@@ -0,0 +1,104 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
+annotated-types==0.7.0
+anyio==4.4.0
+appnope==0.1.4
+asgiref==3.8.1
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.2.0
+backcall==0.2.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+certifi==2024.6.2
+charset-normalizer==3.3.2
+coingecko==0.13
+decorator==5.1.1
+defusedxml==0.7.1
+Deprecated==1.2.14
+docopt==0.6.2
+exceptiongroup==1.2.1
+executing==2.0.1
+fastjsonschema==2.19.1
+frozenlist==1.4.1
+googleapis-common-protos==1.63.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.7
+importlib_metadata==7.1.0
+ipython==8.12.3
+jedi==0.19.1
+Jinja2==3.1.4
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+jupyterlab_pygments==0.3.0
+logfire==0.42.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mistune==3.0.2
+multidict==6.0.5
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+opentelemetry-api==1.25.0
+opentelemetry-exporter-otlp-proto-common==1.25.0
+opentelemetry-exporter-otlp-proto-http==1.25.0
+opentelemetry-instrumentation==0.46b0
+opentelemetry-instrumentation-asgi==0.46b0
+opentelemetry-instrumentation-fastapi==0.46b0
+opentelemetry-instrumentation-httpx==0.46b0
+opentelemetry-instrumentation-redis==0.46b0
+opentelemetry-instrumentation-requests==0.46b0
+opentelemetry-instrumentation-system-metrics==0.46b0
+opentelemetry-proto==1.25.0
+opentelemetry-sdk==1.25.0
+opentelemetry-semantic-conventions==0.46b0
+opentelemetry-util-http==0.46b0
+packaging==24.1
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+pickleshare==0.7.5
+pipdeptree==2.22.0
+pipreqs==0.5.0
+platformdirs==4.2.2
+prompt_toolkit==3.0.47
+protobuf==4.25.3
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pydantic==2.7.4
+pydantic_core==2.18.4
+Pygments==2.18.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-ulid==2.6.0
+pyzmq==26.0.3
+redis==5.0.6
+referencing==0.35.1
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.18.1
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.5
+stack-data==0.6.3
+tinycss2==1.3.0
+tomli==2.0.1
+tornado==6.4.1
+traitlets==5.14.3
+typing_extensions==4.12.2
+ulid==1.1
+upstash-redis==1.1.0
+urllib3==2.2.1
+wcwidth==0.2.13
+webencodings==0.5.1
+wrapt==1.16.0
+yarg==0.1.9
+yarl==1.9.4
+zipp==3.19.2

requirements.txt

@@ -0,0 +1,148 @@
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+annotated-types==0.7.0
+anyio==3.7.1
+appnope==0.1.4
+asgiref==3.8.1
+asttokens==2.4.1
+async-timeout==4.0.3
+asyncer==0.0.2
+attrs==23.2.0
+backcall==0.2.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bleach==6.1.0
+certifi==2024.6.2
+chainlit==1.1.300
+charset-normalizer==3.3.2
+chevron==0.14.0
+click==8.1.7
+coingecko==0.13
+dataclasses-json==0.5.14
+decorator==5.1.1
+defusedxml==0.7.1
+Deprecated==1.2.14
+distro==1.9.0
+docopt==0.6.2
+exceptiongroup==1.2.1
+executing==2.0.1
+fastapi==0.110.3
+fastjsonschema==2.19.1
+filetype==1.2.0
+frozenlist==1.4.1
+googleapis-common-protos==1.63.1
+grpcio==1.64.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.7
+importlib_metadata==7.1.0
+ipython==8.12.3
+jedi==0.19.1
+Jinja2==3.1.4
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+jupyterlab_pygments==0.3.0
+langchain==0.2.4
+langchain-community==0.2.4
+langchain-core==0.2.6
+langchain-text-splitters==0.2.1
+langsmith==0.1.77
+Lazify==0.4.0
+literalai==0.0.604
+logfire==0.42.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+marshmallow==3.21.3
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mistune==3.0.2
+multidict==6.0.5
+mypy-extensions==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+numpy==1.26.4
+openai==1.34.0
+opentelemetry-api==1.25.0
+opentelemetry-exporter-otlp==1.25.0
+opentelemetry-exporter-otlp-proto-common==1.25.0
+opentelemetry-exporter-otlp-proto-grpc==1.25.0
+opentelemetry-exporter-otlp-proto-http==1.25.0
+opentelemetry-instrumentation==0.46b0
+opentelemetry-instrumentation-asgi==0.46b0
+opentelemetry-instrumentation-fastapi==0.46b0
+opentelemetry-instrumentation-httpx==0.46b0
+opentelemetry-instrumentation-redis==0.46b0
+opentelemetry-instrumentation-requests==0.46b0
+opentelemetry-instrumentation-system-metrics==0.46b0
+opentelemetry-proto==1.25.0
+opentelemetry-sdk==1.25.0
+opentelemetry-semantic-conventions==0.46b0
+opentelemetry-util-http==0.46b0
+orjson==3.10.5
+packaging==23.2
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+pickleshare==0.7.5
+pipdeptree==2.22.0
+pipreqs==0.5.0
+platformdirs==4.2.2
+prompt_toolkit==3.0.47
+protobuf==4.25.3
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pydantic==2.7.4
+pydantic_core==2.18.4
+Pygments==2.18.0
+PyJWT==2.8.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-engineio==4.9.1
+python-multipart==0.0.9
+python-socketio==5.11.2
+python-ulid==2.6.0
+PyYAML==6.0.1
+pyzmq==26.0.3
+redis==5.0.6
+referencing==0.35.1
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.18.1
+simple-websocket==1.0.0
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.5
+SQLAlchemy==2.0.30
+stack-data==0.6.3
+starlette==0.37.2
+syncer==2.0.3
+tenacity==8.3.0
+tinycss2==1.3.0
+tomli==2.0.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+ulid==1.1
+upstash-redis==1.1.0
+uptrace==1.24.0
+urllib3==2.2.1
+uvicorn==0.25.0
+watchfiles==0.20.0
+wcwidth==0.2.13
+webencodings==0.5.1
+wrapt==1.16.0
+wsproto==1.2.0
+yarg==0.1.9
+yarl==1.9.4
+zipp==3.19.2

run.py

@@ -0,0 +1,42 @@
+import json
+import pprint
+from src.libs.logger import logger
+
+# from src.data_sources.coingecko import CoinGecko
+# from src.data_sources.cryptocompare import CryptoCompare
+
+
+# cgc = CoinGecko()
+# ccp = CryptoCompare()
+
+# result = cgc.get_coin_price(ids=["bitcoin", "ethereum"], vs_currencies=["usd"])
+# logger.info(f"Got token price: {result}")
+
+# result = cgc.get_coin_data(id="tron")
+# logger.info(f"Got token data: {result}")
+
+# pprint.pprint(result)
+
+# coingecko_instance = get_coingecko_instance()
+# print(coingecko_instance)
+
+# result = coingecko_instance.exchanges.get_list()
+# logger.info(f"Got exchange data: {result}")
+
+# pprint.pprint(result)
+
+
+# result = cgc.get_coin_price(ids=["pepe"], vs_currencies=["usd"])
+# # logger.info(f"Got token data: {result}")
+# pprint.pprint(result)
+
+# result1 = ccp.get_coin_price(ids=["portal"], vs_currencies=["usd"])
+# # logger.info(f"Got token data: {result1}")
+# pprint.pprint(result1)
+
+from ai_functions import AIFunctions
+
+aif = AIFunctions()
+
+result2 = aif.get_asset_price(asset_id="btc", vs_currency="usd")
+pprint.pprint(result2)

src/data_sources/coin_gecko.py

@@ -0,0 +1,71 @@
+import os
+from dotenv import load_dotenv
+from src.databases.redis import REDIS_CACHED
+from src.libs.constants import ONE_HOUR_IN_SECONDS, ONE_MONTH_IN_SECONDS
+from src.libs.logger import logger
+from coingecko import CoinGeckoProClient, CoinGeckoDemoClient
+
+load_dotenv()
+
+redis_cache = REDIS_CACHED
+
+class CoinGecko:
+    def __init__(self, pro_api: bool = False) -> None:
+        if pro_api:
+            self.cgc = CoinGeckoProClient(api_key=os.getenv('COINGECKO_PRO_API_KEY'))
+        else:
+            self.cgc = CoinGeckoDemoClient(api_key=os.getenv('COINGECKO_DEMO_API_KEY'))
+    
+    @redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+    @logger.instrument()
+    def get_coin_price(self, ids: list, vs_currencies: list, cache_ttl: int = None) -> dict:
+        # logger.debug(f"ids: {ids}")
+        # logger.debug(f"vs_currencies: {vs_currencies}")
+
+        result = self.cgc.simple.get_price(ids=ids, vs_currencies=vs_currencies)
+        # logger.debug(f"Result: {result}")
+
+        return result
+    
+    @redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+    @logger.instrument()
+    def get_coin_data(self, id: str, cache_ttl: int = None) -> dict:
+
+        result = self.cgc.coins.get_id(id=id, localization=False, market_data=False, tickers=False, sparkline=False)
+        logger.debug(f"Result: {result}")
+
+        return result
+    
+    @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
+    @logger.instrument()
+    def get_coin_category_data(self, params: dict = None, cache_ttl: int = None) -> dict:
+
+        params = params or {
+            "order": "market_cap_desc",
+        }
+
+        result = self.cgc.categories.get_data(endpoint="coins/categories", params=params)
+        logger.debug(f"Result: {result}")
+
+        return { "categories": result}
+    
+    @redis_cache(ttl=ONE_MONTH_IN_SECONDS)
+    @logger.instrument()
+    def get_exchanges_list(self, cache_ttl: int = None) -> dict:
+
+        # result = self.cgc.exchanges.get(per_page=10)
+        # result = self.cgc.exchanges.get_id(id=id)
+        result = self.cgc.exchanges.get_list()
+        logger.debug(f"Result: {result}")
+
+        return { 'exchanges': result }
+    
+    @redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+    @logger.instrument()
+    def get_exchange_data(self, id: str = None, cache_ttl: int = None) -> dict:
+
+        result = self.cgc.exchanges.get_id(id=id)
+        logger.debug(f"Result: {result}")
+
+        return result
+    

src/data_sources/cryptocompare.py

@@ -0,0 +1,63 @@
+import os
+from dotenv import load_dotenv
+
+import httpx
+from src.libs.helper_functions import convert_to_snakecase
+from src.databases.redis import REDIS_CACHED
+from src.libs.constants import ONE_MINUTE_IN_SECONDS
+from src.libs.constants import CRYPTO_COMPARE_BASE_URL
+from src.libs.logger import logger
+
+
+load_dotenv()
+
+redis_cache = REDIS_CACHED
+
+class CryptoCompare:
+    def __init__(self, base_url: str = None) -> None:
+        self.CRYPTO_COMPARE_BASE_URL = base_url or CRYPTO_COMPARE_BASE_URL
+
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    @logger.instrument()
+    def get_all_coins(self) -> dict:
+        url = f"{self.CRYPTO_COMPARE_BASE_URL}data/all/coinlist"
+        logger.debug(url)
+        params = {"api_key": os.getenv("CRYPTOCOMPARE_API_KEY")}
+        logger.debug(params)
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        logger.debug(headers)
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return convert_to_snakecase(response.json())
+        except httpx.HTTPError as e:
+            print(f"An error occurred while making the request: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    @logger.instrument()
+    def get_coin_price(self, ids: list, vs_currencies: list, cache_ttl: int = None) -> dict:
+        
+        url = f"{self.CRYPTO_COMPARE_BASE_URL}data/pricemulti"
+        # logger.debug(url)
+
+        params = {
+            "api_key": os.getenv("CRYPTOCOMPARE_API_KEY"),
+            "fsyms": ",".join(ids),
+            "tsyms": ",".join(vs_currencies)
+        }
+        # logger.debug(f"{params}")
+
+        headers = {"Content-type":"application/json; charset=UTF-8"}
+        # logger.debug(f"{headers}")
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(url, params=params, headers=headers)
+                response.raise_for_status()  # Raise an exception if the request was unsuccessful
+                return response.json()
+        except httpx.HTTPError as e:
+            logger.debug(f"An error occurred while making the request: {e}")
+            return None

src/data_sources/dexscreener.py

@@ -0,0 +1,107 @@
+from dotenv import load_dotenv
+
+import httpx
+from typing import List, Union, Self
+from src.libs.helper_functions import convert_to_snakecase
+from src.databases.redis import REDIS_CACHED
+from src.libs.constants import ONE_MINUTE_IN_SECONDS
+from src.libs.constants import DEX_SCREENER_BASE_URL
+
+load_dotenv()
+
+redis_cache = REDIS_CACHED
+
+class DexScreener:
+    """
+    A class for interacting with the Dex Screener API.
+
+    Attributes:
+        DEX_SCREENER_BASE_URL (str): The base URL for the Dex Screener API.
+
+    Methods:
+        __init__(self, base_url: str) -> None:
+            Initialize the DexScreener class.
+
+        get_pairs(self, chain_id: str, pair_addresses: str) -> dict:
+
+        get_tokens(self, token_addresses: Union[str, List[str]]) -> dict:
+
+        search_pairs(self, query: str) -> dict:
+    """
+    def __init__(self, base_url: str = None) -> None:
+        self.DEX_SCREENER_BASE_URL = base_url or DEX_SCREENER_BASE_URL
+
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    def get_pairs(self, chain_id: str, pair_addresses: str) -> dict:
+        """
+        This method is used to fetch pair data from Dex Screener API based on the provided chain_id and pair_addresses.
+
+        Parameters:
+        chain_id (str): The ID of the blockchain network.
+        pair_addresses (str): The address(es) of the pair(s) on the blockchain.
+
+        Returns:
+        dict: A dictionary containing the JSON response from the API. Returns None if an error occurs.
+
+        Raises:
+        httpx.HTTPError: If an error occurs while making the request to the API.
+        """
+        url = f"{self.DEX_SCREENER_BASE_URL}pairs/{chain_id}/{pair_addresses}"
+        try:
+            response = httpx.get(url)
+            response.raise_for_status()  # Raise an exception if the request was unsuccessful
+            return convert_to_snakecase(response.json())
+        except httpx.HTTPError as e:
+            print(f"An error occurred while making the request: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    def get_tokens(self, token_addresses: Union[str, List[str]]) -> dict:
+        """
+        This method is used to fetch token data from Dex Screener API based on the provided token_addresses.
+
+        Parameters:
+        token_addresses (Union[str, List[str]]): The address(es) of the token(s) on the blockchain.
+            This parameter can be a single address (str) or a list of addresses (List[str]).
+
+        Returns:
+        dict: A dictionary containing the JSON response from the API. Returns None if an error occurs.
+
+        Raises:
+        httpx.HTTPError: If an error occurs while making the request to the API.
+        """
+        if isinstance(token_addresses, list):
+            token_addresses = ','.join(token_addresses)
+
+        url = f"{self.DEX_SCREENER_BASE_URL}tokens/{token_addresses}"
+        try:
+            response = httpx.get(url)
+            response.raise_for_status()  # Raise an exception if the request was unsuccessful
+            return convert_to_snakecase(response.json())
+        except httpx.HTTPError as e:
+            print(f"An error occurred while making the request: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    def search_pairs(self, query: str) -> dict:
+        """
+        This method is used to search for pairs matching the provided query from Dex Screener API.
+        The query may include pair address, token address, token name, or token symbol.
+
+        Parameters:
+        query (str): The search query.
+
+        Returns:
+        dict: A dictionary containing the JSON response from the API. Returns None if an error occurs.
+
+        Raises:
+        httpx.HTTPError: If an error occurs while making the request to the API.
+        """
+        url = f"{self.DEX_SCREENER_BASE_URL}search/?q={query}"
+        try:
+            response = httpx.get(url)
+            response.raise_for_status()  # Raise an exception if the request was unsuccessful
+            return convert_to_snakecase(response.json())
+        except httpx.HTTPError as e:
+            print(f"An error occurred while making the request: {e}")
+            return None

src/databases/redis.py

@@ -0,0 +1,129 @@
+import json
+import os
+from dotenv import load_dotenv
+from upstash_redis import Redis
+from src.libs.logger import logger
+from src.libs.helper_functions import chunk_data, create_uuid_from_string
+load_dotenv()
+
+url = os.getenv("UPSTASH_REDIS_REST_URL")
+token = os.getenv("UPSTASH_REDIS_REST_TOKEN")
+
+REDIS = Redis(url=url, token=token)
+REDIS_PIPELINE = REDIS.pipeline()
+
+@logger.instrument()
+def args_to_key(*args, **kwargs):
+    """
+    This function generates a unique key based on the provided arguments and keyword arguments.
+
+    Args:
+        *args: A variable number of positional arguments.
+        **kwargs: A variable number of keyword arguments.
+
+    Returns:
+        str: A unique key string generated from the provided arguments and keyword arguments.
+
+    Example:
+        >>> args_to_key("hello", "world", a=1, b=2)
+        'hello_world_a_1_b_2'
+    """
+    params = []
+
+    # TODO: turn these into debugging logs
+    logger.debug(f"args: {args}")
+    logger.debug(f"kwargs: {kwargs}")
+
+    # Append the names of callable arguments to the params list
+    for arg in args:
+        if callable(arg):
+            params.append(arg.__name__)
+
+    # Append the string representations of non-callable arguments and keyword arguments to the params list
+    for arg in args:
+        if not callable(arg):
+            params.append(cast_args_to_string_and_return_first_index(arg))
+
+    for kwarg in kwargs.values():
+        params.append(str(kwarg))
+
+    # Join the elements in the params list using the '_' character as a separator
+    return "_".join(params)
+
+@logger.instrument()
+def REDIS_CACHED(ttl: int = 3600, chunk: bool = False):
+    """
+    This decorator caches the result of a function call in Redis.
+
+    Args:
+        ttl (int): The time-to-live (in seconds) for the cached result. Defaults to 3600 seconds (1 hour).
+        chunk (bool): Whether to chunk the result of the original function call. Defaults to False.
+
+    Returns:
+        A wrapper function that caches the result of the original function call.
+
+    Example:
+        >>> @REDIS_CACHED(ttl=60,  chunk=True)
+        ... def example_function(arg1, arg2):
+        ...     return arg1 + arg2
+        ...
+        ... cached_result = example_function(3, 4)
+        ... print(cached_result)  # Output: 7
+        ... print(cached_result)  # Output: 7 (from cache)
+    """
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            r = REDIS
+
+            cache_key = args_to_key(func, *args, **kwargs)
+            logger.debug(f"Cache key: {cache_key}") # TODO: turn these into debugging logs
+            cache_key = str(create_uuid_from_string(cache_key))
+            logger.debug(f"Cache key: {cache_key}") # TODO: turn these into debugging logs
+
+            # Test if a matching cache key exists
+            cached = r.get(cache_key)
+            if cached:
+                # Found in cache, return it
+                return json.loads(cached)
+
+            # Otherwise, pass everything to the downstream function
+            result = func(*args, **kwargs)
+
+            # Set cache time-to-live duration
+            # Use the default TTL if not provided as an argument
+            cache_ttl = kwargs.get('cache_ttl')
+            ttl_seconds = cache_ttl or kwargs.get('ttl', ttl)
+
+            if chunk:
+                chunked_result = chunk_data(result.data, 100)
+                for i in range(len(chunked_result)):
+                    r.rpush(cache_key, json.dumps(chunked_result[i]))
+                r.pexpire(cache_key, ttl_seconds)
+            else:
+                # Put the result from downstream function into cache, with a TTL
+                # So next call with the same parameters will be handled by the cache
+                r.setex(cache_key, ttl_seconds, result)
+
+            # Return the result transparently
+            return result
+        return wrapper
+    return decorator
+
+@logger.instrument()
+def cast_args_to_string_and_return_first_index(args):
+    """
+    This function takes an argument, casts it to a string, removes '<' and '>', splits it by spaces, and returns the first index.
+
+    Args:
+        args (any): The argument to be processed.
+
+    Returns:
+        str or None: The first index of the argument after processing, or None if the argument is empty.
+
+    Example:
+        >>> cast_args_to_string_and_return_first_index("hello <world>")
+        'hello'
+    """
+    args_str = str(args).strip('<>').split(' ')
+    return args_str[0] if args_str else None
+

src/libs/constants.py

@@ -0,0 +1,61 @@
+"""
+Constants representing the nuumber of seconds in a week.
+"""
+ONE_MONTH_IN_SECONDS : int = 86400 * 30
+
+"""
+Constants representing the nuumber of seconds in a week.
+"""
+ONE_WEEK_IN_SECONDS : int = 604800
+
+"""
+Constants representing the nuumber of seconds in a day.
+"""
+ONE_DAY_IN_SECONDS : int = 86400
+
+"""
+Constant representing the number of seconds in one hour.
+"""
+ONE_HOUR_IN_SECONDS : int = 3600
+
+"""
+Constant representing the number of seconds in one minute.
+"""
+ONE_MINUTE_IN_SECONDS : int = 60
+
+"""
+DEX_SCREENER_BASE_URL : str = "https://api.dexscreener.com/latest/dex/"
+
+This constant is used to represent the base URL for DexScreener API.
+It is defined as "https://api.dexscreener.com/latest/dex/".
+"""
+DEX_SCREENER_BASE_URL : str = "https://api.dexscreener.com/latest/dex/"
+
+"""
+JINA_SEARCH_BASE_ENDPOINT : str = "https://s.jina.ai/"
+
+This constant is used to represent the base endpoint for Jina Search API.
+It is defined as "https://s.jina.ai/".
+"""
+JINA_SEARCH_BASE_ENDPOINT : str = "https://s.jina.ai/"
+
+"""
+JINA_READER_BASE_ENDPOINT : str = "https://r.jina.ai/"
+
+This constant is used to represent the base endpoint for Jina Reader API.
+It is defined as "https://r.jina.ai/".
+"""
+JINA_READER_BASE_ENDPOINT : str = "https://r.jina.ai/"
+
+"""
+CRYPTO_COMPARE_BASE_URL: str = "https://min-api.cryptocompare.com/"
+
+This constant is used to represent the base URL for CryptoCompare API.
+It is defined as "https://min-api.cryptocompare.com/".
+"""
+CRYPTO_COMPARE_BASE_URL: str = "https://min-api.cryptocompare.com/"
+
+
+# TTLs
+
+SEARCH_DATA_TTL : int = ONE_WEEK_IN_SECONDS

src/libs/helper_functions.py

@@ -0,0 +1,132 @@
+import re
+import uuid
+import hashlib
+from ulid import ULID
+
+def chunk_data(data : list | dict, chunk_size: int):
+    """
+    This function takes an array and a chunk size as input, and returns a new array
+    where the original array is divided into smaller chunks of the specified size.
+
+    Parameters:
+    data     (list): The original data to be chunked.
+    chunk_size (int): The size of each chunk.
+
+    Returns:
+    list: A new array containing the chunks of the original array.
+
+    Example:
+    >>> chunk_array([1, 2, 3, 4, 5, 6], 2)
+    [[1, 2], [3, 4], [5, 6]]
+    """
+    is_object = isinstance(data, dict)
+    is_array = isinstance(data, list)
+
+    if not is_object and not is_array:
+        raise TypeError("Data must be a list or a dictionary.")
+    elif is_array:
+        return [data[i:i+chunk_size] for i in range(0, len(data), chunk_size)]
+    elif is_object:
+        items = list(data.items())
+        for i in range(0, len(items), chunk_size):
+            yield dict(items[i:i + chunk_size])
+
+
+def generate_ulid(seed: any = None) -> str:
+    """
+    This function generates a Universally Unique Lexicographically Sortable Identifier (ULID).
+    If a seed is provided, it will be used as the basis for the ULID generation.
+
+    Parameters:
+    seed (any, optional): A value to be used as the basis for the ULID generation. Defaults to None.
+
+    Returns:
+    str: A string representing the generated ULID.
+
+    Example:
+    >>> generate_ulid()
+    '00000000-0001-0100-0000-000000000001'
+    >>> generate_ulid('example_seed')
+    '00000000-0001-0100-0000-000000000002'
+    """
+    if seed is None:
+        ulid = ULID()
+    else:
+        ulid = ULID(seed)
+
+    return ulid.generate()
+
+def create_uuid_from_string(val: str) -> str:
+    """
+    This function takes a string as input and generates a UUID (Universally Unique Identifier)
+    using the input string as the basis for the hash.
+
+    Parameters:
+    val (str): The input string from which the UUID will be generated.
+
+    Returns:
+    str: A string representing the generated UUID.
+
+    Example:
+    >>> create_uuid_from_string('example_string')
+    '00000000-0001-0100-0000-000000000001'
+    """
+    hex_string = hashlib.md5(val.encode("UTF-8")).hexdigest()
+    return str(uuid.UUID(hex=hex_string))
+
+
+def to_snake_case(s):
+    """
+    This function takes a string as input and converts it to snake_case format.
+
+    Parameters:
+    s (str): The input string to be converted to snake_case.
+
+    Returns:
+    str: A string in snake_case format.
+
+    Example:
+    >>> to_snake_case('FirstName')
+    'first_name'
+    """
+    if not s:
+        return ''
+    
+    # Check if the string is in all caps
+    if s.isupper():
+        return s
+    
+    return '_'.join(
+        word.lower() for word in re.findall(r'[A-Z]{2,}(?=[A-Z][a-z]+[0-9]*|\b)|[A-Z]?[a-z]+[0-9]*|[A-Z]|[0-9]+', s)
+    )
+
+def convert_to_snakecase(original_data):
+    """
+    This function takes a dictionary or list of dictionaries as input and converts its keys to snake_case format.
+    If the input is a list of dictionaries, it will recursively convert each nested dictionary.
+
+    Parameters:
+    original_data (dict or list): The input dictionary or list of dictionaries.
+
+    Returns:
+    dict: A new dictionary with keys in snake_case format.
+
+    Example:
+    >>> convert_to_snakecase({'FirstName': 'John', 'LastName': 'Doe'})
+    {'first_name': 'John', 'last_name': 'Doe'}
+    >>> convert_to_snakecase([{'FirstName': 'Jane', 'LastName': 'Smith'}, {'FirstName': 'Bob', 'LastName': 'Johnson'}])
+    [{'first_name': 'Jane', 'last_name': 'Smith'}, {'first_name': 'Bob', 'last_name': 'Johnson'}]
+    """
+    if isinstance(original_data, dict):
+        transformed_dict = {}
+        for k, v in original_data.items():
+            new_key = to_snake_case(k)
+            if isinstance(v, (dict, list)):
+                transformed_dict[new_key] = convert_to_snakecase(v)
+            else:
+                transformed_dict[new_key] = v
+        return transformed_dict
+    elif isinstance(original_data, list):
+        return [convert_to_snakecase(item) for item in original_data]
+    else:
+        raise TypeError("Input must be a dictionary or a list of dictionaries.")

src/libs/logger.py

@@ -0,0 +1,17 @@
+import os
+from dotenv import load_dotenv
+
+import logfire
+
+load_dotenv()
+
+logfire.configure(
+    token=os.getenv('LOGFIRE_TOKEN'),
+    pydantic_plugin=logfire.PydanticPlugin(record='all'),
+    console=logfire.ConsoleOptions(min_log_level= os.getenv('LOG_LEVEL'))
+)
+logfire.instrument_redis()
+logfire.instrument_httpx()
+logfire.instrument_requests()
+
+logger = logfire

src/libs/redis.py

@@ -0,0 +1,14 @@
+keys = {
+    "cryptocompare": {
+        "top_exchanges": "top_exchanges",
+        "all_exchanges": "all_exchanges",
+        "top_assets": "top_assets",
+        "all_assets": "all_assets",
+    },
+    "coingecko": {
+        "top_exchanges": "top_exchanges",
+        "all_exchanges": "all_exchanges",
+        "top_assets": "top_assets",
+        "all_assets": "all_assets",
+    }
+}

src/llms/sourcegraph.py

@@ -0,0 +1,126 @@
+import os
+from dotenv import load_dotenv
+
+from scrapegraphai.graphs import SearchGraph
+from scrapegraphai.graphs import SmartScraperMultiGraph
+from scrapegraphai.graphs import ScriptCreatorGraph
+
+from src.databases.redis import REDIS_CACHED
+from src.libs.constants import ONE_HOUR_IN_SECONDS
+from src.libs.logger import logger
+
+
+load_dotenv()
+
+redis_cache = REDIS_CACHED
+
+default_graph_config: dict = {
+    "openai": {
+        "llm": {
+            "api_key": os.getenv("OPENAI_API_KEY"),
+            "model": "gpt-3.5-turbo",
+            # "model": "gpt-4o",?
+            "temperature": 0,
+        }
+    },
+    "groq": {
+        "llm": {
+            "model": "groq/llama3-70b-8192",
+            "api_key": os.getenv("GROQ_API_KEY"),
+            "temperature": 0
+        },
+        "embeddings": {
+            "model": "ollama/nomic-embed-text",
+            "base_url": "http://localhost:11434",
+        }
+    }
+}
+
+@redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+@logger.instrument()
+def scrape_graph(prompt: str, sources: str | list[str], config: dict | str = None, cache_ttl: int = None) -> str:
+    """
+    This function scrapes the web using a multi-graph approach. It takes a prompt, sources, optional configuration, and optional cache time-to-live.
+
+    Args:
+        prompt (str): The prompt or query to be used for scraping.
+        sources (str | list[str]): The source(s) from which to scrape data. It can be a single string or a list of strings.
+        config (dict | str, optional): The configuration for the scraping process. Defaults to the default "openai" configuration.
+        cache_ttl (int, optional): The time-to-live (in seconds) for the cached results. Defaults to one hour.
+
+    Returns:
+        str: The scraped data as a string.
+    """
+    logger.info({'prompt': prompt, 'sources': sources, 'config': config, 'cache_ttl': cache_ttl})
+
+    config = type(config) == str and default_graph_config[config] or config or default_graph_config["openai"]
+    logger.debug(f"Config: {config}")
+
+    smart_scraper_graph = SmartScraperMultiGraph(
+        prompt=prompt,
+        source=sources,
+        config=config
+    )
+
+    result = smart_scraper_graph.run()
+    logger.debug(f"Result: {result}")
+
+    return result
+
+@redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+@logger.instrument()
+def search_graph(prompt: str, config: dict | str = None, cache_ttl: int = None) -> str:
+    """
+    This function uses a SearchGraph to search the web for relevant information based on the provided prompt.
+
+    Args:
+        prompt (str): The prompt or query to be used for searching.
+        config (dict | str, optional): The configuration for the search process. Defaults to the default "openai" configuration.
+        cache_ttl (int, optional): The time-to-live (in seconds) for the cached results. Defaults to one hour.
+
+    Returns:
+        str: The search results as a string.
+    """
+    logger.info({'prompt': prompt, 'config': config, 'cache_ttl': cache_ttl})
+
+    config = type(config) == str and default_graph_config[config] or config or default_graph_config["openai"]
+    logger.debug(f"Config: {config}")
+
+    search_graph = SearchGraph(
+        prompt=prompt,
+        config=config,
+    )
+
+    result = search_graph.run()
+    logger.debug(f"Result: {result}")
+
+    return result
+
+@redis_cache(ttl=ONE_HOUR_IN_SECONDS)
+@logger.instrument()
+def _create_script_graph(prompt: str, source: str, library: str, config: dict | str = None, cache_ttl: int = None) -> str:
+    """
+    This function creates a ScriptCreatorGraph for generating scripts based on the provided prompt, source, and library.
+
+    Args:
+        prompt (str): The prompt or query to be used for generating the script.
+        source (str): The source from which to generate the script.
+        library (str): The library to be used for generating the script.
+        config (dict | str, optional): The configuration for the script creation process. Defaults to the default "openai" configuration.
+        cache_ttl (int, optional): The time-to-live (in seconds) for the cached results. Defaults to one hour.
+
+    Returns:
+        str: The generated script as a string.
+    """
+    config = type(config) == str and default_graph_config[config] or config or default_graph_config["openai"]
+
+    script_creator_graph = ScriptCreatorGraph(
+        prompt=prompt,
+        source=source,
+        config=config,
+        library=library
+    )
+
+    result = script_creator_graph.run()
+
+    return result

src/requirements.txt

@@ -0,0 +1,8 @@
+auto_mix_prep==0.2.0
+httpx==0.27.0
+logfire==0.42.0
+pyjsonq==1.0.2
+python-dotenv==1.0.1
+scrapegraphai==1.6.0
+ulid==1.1
+upstash_redis==1.1.0

src/search_services/jina_ai.py

@@ -0,0 +1,100 @@
+from dotenv import load_dotenv
+
+import httpx
+import urllib.parse
+from typing import Self
+from src.libs.helper_functions import convert_to_snakecase
+from src.databases.redis import REDIS_CACHED
+from src.libs.constants import ONE_MINUTE_IN_SECONDS
+from src.libs.constants import JINA_READER_BASE_ENDPOINT, JINA_SEARCH_BASE_ENDPOINT, ONE_MINUTE_IN_SECONDS
+
+load_dotenv()
+
+redis_cache = REDIS_CACHED
+
+class JinaAI:
+    """
+    A class for interacting with Jina AI's search and reader services.
+
+    Attributes:
+        JINA_SEARCH_BASE_ENDPOINT (str): The base URL for the Jina AI search service.
+        JINA_READER_BASE_ENDPOINT (str): The base URL for the Jina AI reader service.
+
+    Methods:
+        __init__(self, search_base_url: str = None, reader_base_url: str = None) -> None:
+            Initialize the JinaAI instance with optional search and reader base URLs.
+
+        search_web_with_jina(self, search_query: str) -> dict | None:
+            Search the web using Jina AI and return the search results as a dictionary.
+
+        read_website_with_jina(self, website_url: str) -> dict | None:
+            Read a website using Jina AI and return the website content as a dictionary.
+    """
+    def __init__(self, search_base_url: str = None, reader_base_url: str = None) -> None:
+        self.JINA_SEARCH_BASE_ENDPOINT = search_base_url or JINA_SEARCH_BASE_ENDPOINT
+        self.JINA_READER_BASE_ENDPOINT = reader_base_url or JINA_READER_BASE_ENDPOINT
+
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    def search_web_with_jina(self, search_query: str) -> dict | None:
+        """
+        Search the web using Jina AI.
+
+        Args:
+            search_query (str): The query to be searched on the web.
+
+        Returns:
+            dict | None: A dictionary containing the search results if successful, otherwise None.
+
+        Raises:
+            httpx.HTTPError: If an HTTP error occurs during the request.
+
+        Usage:
+            jina_ai_instance.search_web_with_jina(search_query)
+
+        Example:
+            jina_ai_instance.search_web_with_jina("example search query")
+        """
+        url = self.JINA_SEARCH_BASE_ENDPOINT
+        encoded_search_query = urllib.parse.quote(search_query)
+        headers = {"Accept": "application/json"}
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(f"{url}{encoded_search_query}", headers=headers)
+                response.raise_for_status()
+                return convert_to_snakecase(response.json())
+        except httpx.HTTPError as e:
+            print(f"An error occurred: {e}")
+            return None
+        
+    @redis_cache(ttl=ONE_MINUTE_IN_SECONDS)
+    def read_website_with_jina(self, website_url: str) -> dict | None:
+        """
+        Read a website using Jina AI.
+
+        Args:
+            website_url (str): The URL of the website to be read.
+
+        Returns:
+            dict | None: A dictionary containing the content of the website if successful, otherwise None.
+
+        Raises:
+            httpx.HTTPError: If an HTTP error occurs during the request.
+
+        Usage:
+            jina_ai_instance.read_website_with_jina(website_url)
+
+        Example:
+            jina_ai_instance.read_website_with_jina("https://example.com")
+        """
+        url = self.JINA_READER_BASE_ENDPOINT
+        headers = {"Accept": "application/json"}
+
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                response = client.get(f"{url}{website_url}", headers=headers)
+                response.raise_for_status()
+                return convert_to_snakecase(response.json())
+        except httpx.HTTPError as e:
+            print(f"An error occurred: {e}")
+            return None

src/tools/crypto_coin_price_tool.py

@@ -0,0 +1,60 @@
+from pydantic import BaseModel, Field
+from typing import Dict, Optional, Type
+
+from src.data_sources.coin_gecko import CoinGecko
+from src.data_sources.cryptocompare import CryptoCompare
+
+from langchain.tools.base import BaseTool
+
+class CryptoCoinPrice(BaseModel):
+    """Represents the prices of a coin in various currencies."""
+    prices: Dict[str, float] = Field(..., description="Prices in various currencies")
+
+class CryptoCoinPriceData(BaseModel):
+    """Encapsulates both CoinGecko and CryptoCompare price data."""
+    coingecko_price: Dict[str, CryptoCoinPrice] = Field(..., description="CoinGecko prices for various coins")
+    crypto_compare_price: Dict[str, CryptoCoinPrice] = Field(..., description="CryptoCompare prices for various coins")
+
+class PriceInput(BaseModel):
+    coin_id: str = Field(..., description="The ID of the cryptocurrency coin to retrieve prices for")
+    vs_currency: str = Field("usd", description="The currency to compare against")
+
+class CryptoCoinPriceOutput(BaseModel):
+    price_data: CryptoCoinPriceData
+
+class CryptoCoinPriceTool(BaseTool):
+    name = "CryptoCoinPriceTool"
+    description = "Fetches price data for a given cryptocurrency coin from CoinGecko and CryptoCompare"
+    args_schema: Type[BaseModel] = PriceInput
+    return_direct: bool = True
+    
+    def __init__(self, id: Optional[str] = None):
+        self.id = id
+        self.coingecko = CoinGecko()
+        self.crypto_compare = CryptoCompare()
+
+    def _run(self, coin_id: str, vs_currency: str = "usd") -> CryptoCoinPriceData:
+        coingecko_price_data = self.coingecko.get_coin_price(ids=[coin_id], vs_currencies=[vs_currency])
+        crypto_compare_price_data = self.crypto_compare.get_coin_price(ids=[coin_id], vs_currencies=[vs_currency])
+
+        coingecko_price = {}
+        crypto_compare_price = {}
+
+        if coin_id in coingecko_price_data:
+            coingecko_price[coin_id] = CryptoCoinPrice(prices=coingecko_price_data[coin_id])
+        else:
+            print(f"Warning: CoinGecko data for {coin_id} not found.")
+
+        if coin_id.upper() in crypto_compare_price_data:
+            crypto_compare_price[coin_id] = CryptoCoinPrice(prices=crypto_compare_price_data[coin_id.upper()])
+        else:
+            print(f"Warning: CryptoCompare data for {coin_id} not found.")
+
+        return CryptoCoinPriceData(
+            coingecko_price=coingecko_price,
+            crypto_compare_price=crypto_compare_price
+        )
+
+    def __call__(self, inputs: PriceInput) -> CryptoCoinPriceOutput:
+        price_data = self._run(inputs.coin_id, inputs.vs_currency)
+        return CryptoCoinPriceOutput(price_data=price_data)

src/xbt-core.py

@@ -0,0 +1,43 @@
+import os
+import httpx
+import urllib.parse
+from typing import Self
+from pyjsonq import JsonQ
+from dotenv import load_dotenv
+
+load_dotenv()
+
+JINA_SEARCH_BASE_ENDPOINT = "r.jina.ai"
+JINA_READER_BASE_ENDPOINT = "s.jina.ai"
+
+class XBTCore:
+    def __init__(self) -> None:
+        self.value = 0
+        self.JINA_SEARCH_BASE_ENDPOINT = "s.jina.ai"
+        self.JINA_READER_BASE_ENDPOINT = "r.jina.ai"
+
+    def search_web_with_jina(self, search_query: str=False) -> Self:
+        url = self.JINA_SEARCH_BASE_ENDPOINT
+        encoded_search_query = urllib.parse.quote(search_query)
+
+        try:
+            with httpx.Client() as client:
+                response = client.get(f"{url}/{encoded_search_query}")
+                response.raise_for_status()
+                return response.json()
+        except httpx.HTTPError as e:
+            print(f"An error occurred: {e}")
+            return None
+        
+    def read_website_with_jina(self, website_url: str=False) -> Self:
+        url = self.JINA_READER_BASE_ENDPOINT
+
+        try:
+            with httpx.Client() as client:
+                response = client.get(f"{url}/{website_url}")
+                response.raise_for_status()
+                return response.json()
+        except httpx.HTTPError as e:
+            print(f"An error occurred: {e}")
+            return None
+