Upload 235 files

litellm/__init__.py

@@ -0,0 +1,557 @@
+### INIT VARIABLES ###
+import threading, requests
+from typing import Callable, List, Optional, Dict, Union, Any
+from litellm.caching import Cache
+from litellm._logging import set_verbose
+from litellm.proxy._types import KeyManagementSystem
+import httpx
+
+input_callback: List[Union[str, Callable]] = []
+success_callback: List[Union[str, Callable]] = []
+failure_callback: List[Union[str, Callable]] = []
+callbacks: List[Callable] = []
+_async_input_callback: List[
+    Callable
+] = []  # internal variable - async custom callbacks are routed here.
+_async_success_callback: List[
+    Union[str, Callable]
+] = []  # internal variable - async custom callbacks are routed here.
+_async_failure_callback: List[
+    Callable
+] = []  # internal variable - async custom callbacks are routed here.
+pre_call_rules: List[Callable] = []
+post_call_rules: List[Callable] = []
+email: Optional[
+    str
+] = None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+token: Optional[
+    str
+] = None  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+telemetry = True
+max_tokens = 256  # OpenAI Defaults
+drop_params = False
+retry = True
+api_key: Optional[str] = None
+openai_key: Optional[str] = None
+azure_key: Optional[str] = None
+anthropic_key: Optional[str] = None
+replicate_key: Optional[str] = None
+cohere_key: Optional[str] = None
+maritalk_key: Optional[str] = None
+ai21_key: Optional[str] = None
+openrouter_key: Optional[str] = None
+huggingface_key: Optional[str] = None
+vertex_project: Optional[str] = None
+vertex_location: Optional[str] = None
+togetherai_api_key: Optional[str] = None
+cloudflare_api_key: Optional[str] = None
+baseten_key: Optional[str] = None
+aleph_alpha_key: Optional[str] = None
+nlp_cloud_key: Optional[str] = None
+use_client: bool = False
+logging: bool = True
+caching: bool = False  # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+caching_with_models: bool = False  # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
+cache: Optional[
+    Cache
+] = None  # cache object <- use this - https://docs.litellm.ai/docs/caching
+model_alias_map: Dict[str, str] = {}
+model_group_alias_map: Dict[str, str] = {}
+max_budget: float = 0.0  # set the max budget across all providers
+_openai_completion_params = [
+    "functions",
+    "function_call",
+    "temperature",
+    "temperature",
+    "top_p",
+    "n",
+    "stream",
+    "stop",
+    "max_tokens",
+    "presence_penalty",
+    "frequency_penalty",
+    "logit_bias",
+    "user",
+    "request_timeout",
+    "api_base",
+    "api_version",
+    "api_key",
+    "deployment_id",
+    "organization",
+    "base_url",
+    "default_headers",
+    "timeout",
+    "response_format",
+    "seed",
+    "tools",
+    "tool_choice",
+    "max_retries",
+]
+_litellm_completion_params = [
+    "metadata",
+    "acompletion",
+    "caching",
+    "mock_response",
+    "api_key",
+    "api_version",
+    "api_base",
+    "force_timeout",
+    "logger_fn",
+    "verbose",
+    "custom_llm_provider",
+    "litellm_logging_obj",
+    "litellm_call_id",
+    "use_client",
+    "id",
+    "fallbacks",
+    "azure",
+    "headers",
+    "model_list",
+    "num_retries",
+    "context_window_fallback_dict",
+    "roles",
+    "final_prompt_value",
+    "bos_token",
+    "eos_token",
+    "request_timeout",
+    "complete_response",
+    "self",
+    "client",
+    "rpm",
+    "tpm",
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "hf_model_name",
+    "model_info",
+    "proxy_server_request",
+    "preset_cache_key",
+]
+_current_cost = 0  # private variable, used if max budget is set
+error_logs: Dict = {}
+add_function_to_prompt: bool = False  # if function calling not supported by api, append function call details to system prompt
+client_session: Optional[httpx.Client] = None
+aclient_session: Optional[httpx.AsyncClient] = None
+model_fallbacks: Optional[List] = None  # Deprecated for 'litellm.fallbacks'
+model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+suppress_debug_info = False
+dynamodb_table_name: Optional[str] = None
+s3_callback_params: Optional[Dict] = None
+#### RELIABILITY ####
+request_timeout: Optional[float] = 6000
+num_retries: Optional[int] = None  # per model endpoint
+fallbacks: Optional[List] = None
+context_window_fallbacks: Optional[List] = None
+allowed_fails: int = 0
+num_retries_per_request: Optional[
+    int
+] = None  # for the request overall (incl. fallbacks + model retries)
+####### SECRET MANAGERS #####################
+secret_manager_client: Optional[
+    Any
+] = None  # list of instantiated key management clients - e.g. azure kv, infisical, etc.
+_google_kms_resource_name: Optional[str] = None
+_key_management_system: Optional[KeyManagementSystem] = None
+#############################################
+
+
+def get_model_cost_map(url: str):
+    try:
+        with requests.get(
+            url, timeout=5
+        ) as response:  # set a 5 second timeout for the get request
+            response.raise_for_status()  # Raise an exception if the request is unsuccessful
+            content = response.json()
+            return content
+    except Exception as e:
+        import importlib.resources
+        import json
+
+        with importlib.resources.open_text(
+            "litellm", "model_prices_and_context_window_backup.json"
+        ) as f:
+            content = json.load(f)
+            return content
+
+
+model_cost = get_model_cost_map(url=model_cost_map_url)
+custom_prompt_dict: Dict[str, dict] = {}
+
+
+####### THREAD-SPECIFIC DATA ###################
+class MyLocal(threading.local):
+    def __init__(self):
+        self.user = "Hello World"
+
+
+_thread_context = MyLocal()
+
+
+def identify(event_details):
+    # Store user in thread local data
+    if "user" in event_details:
+        _thread_context.user = event_details["user"]
+
+
+####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.
+api_base = None
+headers = None
+api_version = None
+organization = None
+config_path = None
+####### COMPLETION MODELS ###################
+open_ai_chat_completion_models: List = []
+open_ai_text_completion_models: List = []
+cohere_models: List = []
+anthropic_models: List = []
+openrouter_models: List = []
+vertex_language_models: List = []
+vertex_vision_models: List = []
+vertex_chat_models: List = []
+vertex_code_chat_models: List = []
+vertex_text_models: List = []
+vertex_code_text_models: List = []
+ai21_models: List = []
+nlp_cloud_models: List = []
+aleph_alpha_models: List = []
+bedrock_models: List = []
+deepinfra_models: List = []
+perplexity_models: List = []
+for key, value in model_cost.items():
+    if value.get("litellm_provider") == "openai":
+        open_ai_chat_completion_models.append(key)
+    elif value.get("litellm_provider") == "text-completion-openai":
+        open_ai_text_completion_models.append(key)
+    elif value.get("litellm_provider") == "cohere":
+        cohere_models.append(key)
+    elif value.get("litellm_provider") == "anthropic":
+        anthropic_models.append(key)
+    elif value.get("litellm_provider") == "openrouter":
+        openrouter_models.append(key)
+    elif value.get("litellm_provider") == "vertex_ai-text-models":
+        vertex_text_models.append(key)
+    elif value.get("litellm_provider") == "vertex_ai-code-text-models":
+        vertex_code_text_models.append(key)
+    elif value.get("litellm_provider") == "vertex_ai-language-models":
+        vertex_language_models.append(key)
+    elif value.get("litellm_provider") == "vertex_ai-vision-models":
+        vertex_vision_models.append(key)
+    elif value.get("litellm_provider") == "vertex_ai-chat-models":
+        vertex_chat_models.append(key)
+    elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
+        vertex_code_chat_models.append(key)
+    elif value.get("litellm_provider") == "ai21":
+        ai21_models.append(key)
+    elif value.get("litellm_provider") == "nlp_cloud":
+        nlp_cloud_models.append(key)
+    elif value.get("litellm_provider") == "aleph_alpha":
+        aleph_alpha_models.append(key)
+    elif value.get("litellm_provider") == "bedrock":
+        bedrock_models.append(key)
+    elif value.get("litellm_provider") == "deepinfra":
+        deepinfra_models.append(key)
+    elif value.get("litellm_provider") == "perplexity":
+        perplexity_models.append(key)
+
+# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary
+openai_compatible_endpoints: List = [
+    "api.perplexity.ai",
+    "api.endpoints.anyscale.com/v1",
+    "api.deepinfra.com/v1/openai",
+    "api.mistral.ai/v1",
+]
+
+# this is maintained for Exception Mapping
+openai_compatible_providers: List = [
+    "anyscale",
+    "mistral",
+    "deepinfra",
+    "perplexity",
+    "xinference",
+]
+
+
+# well supported replicate llms
+replicate_models: List = [
+    # llama replicate supported LLMs
+    "replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
+    "a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
+    "meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db",
+    # Vicuna
+    "replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b",
+    "joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe",
+    # Flan T-5
+    "daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f"
+    # Others
+    "replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
+    "replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
+]
+
+huggingface_models: List = [
+    "meta-llama/Llama-2-7b-hf",
+    "meta-llama/Llama-2-7b-chat-hf",
+    "meta-llama/Llama-2-13b-hf",
+    "meta-llama/Llama-2-13b-chat-hf",
+    "meta-llama/Llama-2-70b-hf",
+    "meta-llama/Llama-2-70b-chat-hf",
+    "meta-llama/Llama-2-7b",
+    "meta-llama/Llama-2-7b-chat",
+    "meta-llama/Llama-2-13b",
+    "meta-llama/Llama-2-13b-chat",
+    "meta-llama/Llama-2-70b",
+    "meta-llama/Llama-2-70b-chat",
+]  # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
+
+together_ai_models: List = [
+    # llama llms - chat
+    "togethercomputer/llama-2-70b-chat",
+    # llama llms - language / instruct
+    "togethercomputer/llama-2-70b",
+    "togethercomputer/LLaMA-2-7B-32K",
+    "togethercomputer/Llama-2-7B-32K-Instruct",
+    "togethercomputer/llama-2-7b",
+    # falcon llms
+    "togethercomputer/falcon-40b-instruct",
+    "togethercomputer/falcon-7b-instruct",
+    # alpaca
+    "togethercomputer/alpaca-7b",
+    # chat llms
+    "HuggingFaceH4/starchat-alpha",
+    # code llms
+    "togethercomputer/CodeLlama-34b",
+    "togethercomputer/CodeLlama-34b-Instruct",
+    "togethercomputer/CodeLlama-34b-Python",
+    "defog/sqlcoder",
+    "NumbersStation/nsql-llama-2-7B",
+    "WizardLM/WizardCoder-15B-V1.0",
+    "WizardLM/WizardCoder-Python-34B-V1.0",
+    # language llms
+    "NousResearch/Nous-Hermes-Llama2-13b",
+    "Austism/chronos-hermes-13b",
+    "upstage/SOLAR-0-70b-16bit",
+    "WizardLM/WizardLM-70B-V1.0",
+]  # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
+
+
+baseten_models: List = [
+    "qvv0xeq",
+    "q841o8w",
+    "31dxrj3",
+]  # FALCON 7B  # WizardLM  # Mosaic ML
+
+
+# used for Cost Tracking & Token counting
+# https://azure.microsoft.com/en-in/pricing/details/cognitive-services/openai-service/
+# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting
+azure_llms = {
+    "gpt-35-turbo": "azure/gpt-35-turbo",
+    "gpt-35-turbo-16k": "azure/gpt-35-turbo-16k",
+    "gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct",
+}
+
+azure_embedding_models = {
+    "ada": "azure/ada",
+}
+
+petals_models = [
+    "petals-team/StableBeluga2",
+]
+
+ollama_models = ["llama2"]
+
+maritalk_models = ["maritalk"]
+
+model_list = (
+    open_ai_chat_completion_models
+    + open_ai_text_completion_models
+    + cohere_models
+    + anthropic_models
+    + replicate_models
+    + openrouter_models
+    + huggingface_models
+    + vertex_chat_models
+    + vertex_text_models
+    + ai21_models
+    + together_ai_models
+    + baseten_models
+    + aleph_alpha_models
+    + nlp_cloud_models
+    + ollama_models
+    + bedrock_models
+    + deepinfra_models
+    + perplexity_models
+    + maritalk_models
+)
+
+provider_list: List = [
+    "openai",
+    "custom_openai",
+    "text-completion-openai",
+    "cohere",
+    "anthropic",
+    "replicate",
+    "huggingface",
+    "together_ai",
+    "openrouter",
+    "vertex_ai",
+    "palm",
+    "gemini",
+    "ai21",
+    "baseten",
+    "azure",
+    "sagemaker",
+    "bedrock",
+    "vllm",
+    "nlp_cloud",
+    "petals",
+    "oobabooga",
+    "ollama",
+    "ollama_chat",
+    "deepinfra",
+    "perplexity",
+    "anyscale",
+    "mistral",
+    "maritalk",
+    "voyage",
+    "cloudflare",
+    "xinference",
+    "custom",  # custom apis
+]
+
+models_by_provider: dict = {
+    "openai": open_ai_chat_completion_models + open_ai_text_completion_models,
+    "cohere": cohere_models,
+    "anthropic": anthropic_models,
+    "replicate": replicate_models,
+    "huggingface": huggingface_models,
+    "together_ai": together_ai_models,
+    "baseten": baseten_models,
+    "openrouter": openrouter_models,
+    "vertex_ai": vertex_chat_models + vertex_text_models,
+    "ai21": ai21_models,
+    "bedrock": bedrock_models,
+    "petals": petals_models,
+    "ollama": ollama_models,
+    "deepinfra": deepinfra_models,
+    "perplexity": perplexity_models,
+    "maritalk": maritalk_models,
+}
+
+# mapping for those models which have larger equivalents
+longer_context_model_fallback_dict: dict = {
+    # openai chat completion models
+    "gpt-3.5-turbo": "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-0301": "gpt-3.5-turbo-16k-0301",
+    "gpt-3.5-turbo-0613": "gpt-3.5-turbo-16k-0613",
+    "gpt-4": "gpt-4-32k",
+    "gpt-4-0314": "gpt-4-32k-0314",
+    "gpt-4-0613": "gpt-4-32k-0613",
+    # anthropic
+    "claude-instant-1": "claude-2",
+    "claude-instant-1.2": "claude-2",
+    # vertexai
+    "chat-bison": "chat-bison-32k",
+    "chat-bison@001": "chat-bison-32k",
+    "codechat-bison": "codechat-bison-32k",
+    "codechat-bison@001": "codechat-bison-32k",
+    # openrouter
+    "openrouter/openai/gpt-3.5-turbo": "openrouter/openai/gpt-3.5-turbo-16k",
+    "openrouter/anthropic/claude-instant-v1": "openrouter/anthropic/claude-2",
+}
+
+####### EMBEDDING MODELS ###################
+open_ai_embedding_models: List = ["text-embedding-ada-002"]
+cohere_embedding_models: List = [
+    "embed-english-v3.0",
+    "embed-english-light-v3.0",
+    "embed-multilingual-v3.0",
+    "embed-english-v2.0",
+    "embed-english-light-v2.0",
+    "embed-multilingual-v2.0",
+]
+bedrock_embedding_models: List = [
+    "amazon.titan-embed-text-v1",
+    "cohere.embed-english-v3",
+    "cohere.embed-multilingual-v3",
+]
+
+all_embedding_models = (
+    open_ai_embedding_models + cohere_embedding_models + bedrock_embedding_models
+)
+
+####### IMAGE GENERATION MODELS ###################
+openai_image_generation_models = ["dall-e-2", "dall-e-3"]
+
+
+from .timeout import timeout
+from .utils import (
+    client,
+    exception_type,
+    get_optional_params,
+    modify_integration,
+    token_counter,
+    cost_per_token,
+    completion_cost,
+    get_litellm_params,
+    Logging,
+    acreate,
+    get_model_list,
+    get_max_tokens,
+    get_model_info,
+    register_prompt_template,
+    validate_environment,
+    check_valid_key,
+    get_llm_provider,
+    register_model,
+    encode,
+    decode,
+    _calculate_retry_after,
+    _should_retry,
+    get_secret,
+)
+from .llms.huggingface_restapi import HuggingfaceConfig
+from .llms.anthropic import AnthropicConfig
+from .llms.replicate import ReplicateConfig
+from .llms.cohere import CohereConfig
+from .llms.ai21 import AI21Config
+from .llms.together_ai import TogetherAIConfig
+from .llms.cloudflare import CloudflareConfig
+from .llms.palm import PalmConfig
+from .llms.gemini import GeminiConfig
+from .llms.nlp_cloud import NLPCloudConfig
+from .llms.aleph_alpha import AlephAlphaConfig
+from .llms.petals import PetalsConfig
+from .llms.vertex_ai import VertexAIConfig
+from .llms.sagemaker import SagemakerConfig
+from .llms.ollama import OllamaConfig
+from .llms.maritalk import MaritTalkConfig
+from .llms.bedrock import (
+    AmazonTitanConfig,
+    AmazonAI21Config,
+    AmazonAnthropicConfig,
+    AmazonCohereConfig,
+    AmazonLlamaConfig,
+)
+from .llms.openai import OpenAIConfig, OpenAITextCompletionConfig
+from .llms.azure import AzureOpenAIConfig, AzureOpenAIError
+from .main import *  # type: ignore
+from .integrations import *
+from .exceptions import (
+    AuthenticationError,
+    InvalidRequestError,
+    BadRequestError,
+    NotFoundError,
+    RateLimitError,
+    ServiceUnavailableError,
+    OpenAIError,
+    ContextWindowExceededError,
+    ContentPolicyViolationError,
+    BudgetExceededError,
+    APIError,
+    Timeout,
+    APIConnectionError,
+    APIResponseValidationError,
+    UnprocessableEntityError,
+)
+from .budget_manager import BudgetManager
+from .proxy.proxy_cli import run_server
+from .router import Router

litellm/_logging.py

@@ -0,0 +1,30 @@
+import logging
+
+set_verbose = False
+
+# Create a handler for the logger (you may need to adapt this based on your needs)
+handler = logging.StreamHandler()
+handler.setLevel(logging.DEBUG)
+
+# Create a formatter and set it for the handler
+
+formatter = logging.Formatter("\033[92m%(name)s - %(levelname)s\033[0m: %(message)s")
+
+handler.setFormatter(formatter)
+
+
+def print_verbose(print_statement):
+    try:
+        if set_verbose:
+            print(print_statement)  # noqa
+    except:
+        pass
+
+
+verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
+verbose_router_logger = logging.getLogger("LiteLLM Router")
+verbose_logger = logging.getLogger("LiteLLM")
+
+# Add the handler to the logger
+verbose_router_logger.addHandler(handler)
+verbose_proxy_logger.addHandler(handler)

litellm/_redis.py

@@ -0,0 +1,93 @@
+# +-----------------------------------------------+
+# |                                               |
+# |           Give Feedback / Get Help            |
+# | https://github.com/BerriAI/litellm/issues/new |
+# |                                               |
+# +-----------------------------------------------+
+#
+#  Thank you users! We ❤️ you! - Krrish & Ishaan
+
+# s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
+import os
+import inspect
+import redis, litellm
+from typing import List, Optional
+
+
+def _get_redis_kwargs():
+    arg_spec = inspect.getfullargspec(redis.Redis)
+
+    # Only allow primitive arguments
+    exclude_args = {
+        "self",
+        "connection_pool",
+        "retry",
+    }
+
+    include_args = ["url"]
+
+    available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
+
+    return available_args
+
+
+def _get_redis_env_kwarg_mapping():
+    PREFIX = "REDIS_"
+
+    return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs()}
+
+
+def _redis_kwargs_from_environment():
+    mapping = _get_redis_env_kwarg_mapping()
+
+    return_dict = {}
+    for k, v in mapping.items():
+        value = litellm.get_secret(k, default_value=None)  # check os.environ/key vault
+        if value is not None:
+            return_dict[v] = value
+    return return_dict
+
+
+def get_redis_url_from_environment():
+    if "REDIS_URL" in os.environ:
+        return os.environ["REDIS_URL"]
+
+    if "REDIS_HOST" not in os.environ or "REDIS_PORT" not in os.environ:
+        raise ValueError(
+            "Either 'REDIS_URL' or both 'REDIS_HOST' and 'REDIS_PORT' must be specified for Redis."
+        )
+
+    if "REDIS_PASSWORD" in os.environ:
+        redis_password = f":{os.environ['REDIS_PASSWORD']}@"
+    else:
+        redis_password = ""
+
+    return (
+        f"redis://{redis_password}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
+    )
+
+
+def get_redis_client(**env_overrides):
+    ### check if "os.environ/<key-name>" passed in
+    for k, v in env_overrides.items():
+        if isinstance(v, str) and v.startswith("os.environ/"):
+            v = v.replace("os.environ/", "")
+            value = litellm.get_secret(v)
+            env_overrides[k] = value
+
+    redis_kwargs = {
+        **_redis_kwargs_from_environment(),
+        **env_overrides,
+    }
+
+    if "url" in redis_kwargs and redis_kwargs["url"] is not None:
+        redis_kwargs.pop("host", None)
+        redis_kwargs.pop("port", None)
+        redis_kwargs.pop("db", None)
+        redis_kwargs.pop("password", None)
+
+        return redis.Redis.from_url(**redis_kwargs)
+    elif "host" not in redis_kwargs or redis_kwargs["host"] is None:
+        raise ValueError("Either 'host' or 'url' must be specified for redis.")
+    litellm.print_verbose(f"redis_kwargs: {redis_kwargs}")
+    return redis.Redis(**redis_kwargs)

litellm/_version.py

@@ -0,0 +1,6 @@
+import importlib_metadata
+
+try:
+    version = importlib_metadata.version("litellm")
+except:
+    pass

litellm/budget_manager.py

@@ -0,0 +1,206 @@
+import os, json, time
+import litellm
+from litellm.utils import ModelResponse
+import requests, threading
+from typing import Optional, Union, Literal
+
+
+class BudgetManager:
+    def __init__(
+        self,
+        project_name: str,
+        client_type: str = "local",
+        api_base: Optional[str] = None,
+    ):
+        self.client_type = client_type
+        self.project_name = project_name
+        self.api_base = api_base or "https://api.litellm.ai"
+        ## load the data or init the initial dictionaries
+        self.load_data()
+
+    def print_verbose(self, print_statement):
+        try:
+            if litellm.set_verbose:
+                import logging
+
+                logging.info(print_statement)
+        except:
+            pass
+
+    def load_data(self):
+        if self.client_type == "local":
+            # Check if user dict file exists
+            if os.path.isfile("user_cost.json"):
+                # Load the user dict
+                with open("user_cost.json", "r") as json_file:
+                    self.user_dict = json.load(json_file)
+            else:
+                self.print_verbose("User Dictionary not found!")
+                self.user_dict = {}
+            self.print_verbose(f"user dict from local: {self.user_dict}")
+        elif self.client_type == "hosted":
+            # Load the user_dict from hosted db
+            url = self.api_base + "/get_budget"
+            headers = {"Content-Type": "application/json"}
+            data = {"project_name": self.project_name}
+            response = requests.post(url, headers=headers, json=data)
+            response = response.json()
+            if response["status"] == "error":
+                self.user_dict = (
+                    {}
+                )  # assume this means the user dict hasn't been stored yet
+            else:
+                self.user_dict = response["data"]
+
+    def create_budget(
+        self,
+        total_budget: float,
+        user: str,
+        duration: Optional[Literal["daily", "weekly", "monthly", "yearly"]] = None,
+        created_at: float = time.time(),
+    ):
+        self.user_dict[user] = {"total_budget": total_budget}
+        if duration is None:
+            return self.user_dict[user]
+
+        if duration == "daily":
+            duration_in_days = 1
+        elif duration == "weekly":
+            duration_in_days = 7
+        elif duration == "monthly":
+            duration_in_days = 28
+        elif duration == "yearly":
+            duration_in_days = 365
+        else:
+            raise ValueError(
+                """duration needs to be one of ["daily", "weekly", "monthly", "yearly"]"""
+            )
+        self.user_dict[user] = {
+            "total_budget": total_budget,
+            "duration": duration_in_days,
+            "created_at": created_at,
+            "last_updated_at": created_at,
+        }
+        self._save_data_thread()  # [Non-Blocking] Update persistent storage without blocking execution
+        return self.user_dict[user]
+
+    def projected_cost(self, model: str, messages: list, user: str):
+        text = "".join(message["content"] for message in messages)
+        prompt_tokens = litellm.token_counter(model=model, text=text)
+        prompt_cost, _ = litellm.cost_per_token(
+            model=model, prompt_tokens=prompt_tokens, completion_tokens=0
+        )
+        current_cost = self.user_dict[user].get("current_cost", 0)
+        projected_cost = prompt_cost + current_cost
+        return projected_cost
+
+    def get_total_budget(self, user: str):
+        return self.user_dict[user]["total_budget"]
+
+    def update_cost(
+        self,
+        user: str,
+        completion_obj: Optional[ModelResponse] = None,
+        model: Optional[str] = None,
+        input_text: Optional[str] = None,
+        output_text: Optional[str] = None,
+    ):
+        if model and input_text and output_text:
+            prompt_tokens = litellm.token_counter(
+                model=model, messages=[{"role": "user", "content": input_text}]
+            )
+            completion_tokens = litellm.token_counter(
+                model=model, messages=[{"role": "user", "content": output_text}]
+            )
+            (
+                prompt_tokens_cost_usd_dollar,
+                completion_tokens_cost_usd_dollar,
+            ) = litellm.cost_per_token(
+                model=model,
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+            )
+            cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
+        elif completion_obj:
+            cost = litellm.completion_cost(completion_response=completion_obj)
+            model = completion_obj[
+                "model"
+            ]  # if this throws an error try, model = completion_obj['model']
+        else:
+            raise ValueError(
+                "Either a chat completion object or the text response needs to be passed in. Learn more - https://docs.litellm.ai/docs/budget_manager"
+            )
+
+        self.user_dict[user]["current_cost"] = cost + self.user_dict[user].get(
+            "current_cost", 0
+        )
+        if "model_cost" in self.user_dict[user]:
+            self.user_dict[user]["model_cost"][model] = cost + self.user_dict[user][
+                "model_cost"
+            ].get(model, 0)
+        else:
+            self.user_dict[user]["model_cost"] = {model: cost}
+
+        self._save_data_thread()  # [Non-Blocking] Update persistent storage without blocking execution
+        return {"user": self.user_dict[user]}
+
+    def get_current_cost(self, user):
+        return self.user_dict[user].get("current_cost", 0)
+
+    def get_model_cost(self, user):
+        return self.user_dict[user].get("model_cost", 0)
+
+    def is_valid_user(self, user: str) -> bool:
+        return user in self.user_dict
+
+    def get_users(self):
+        return list(self.user_dict.keys())
+
+    def reset_cost(self, user):
+        self.user_dict[user]["current_cost"] = 0
+        self.user_dict[user]["model_cost"] = {}
+        return {"user": self.user_dict[user]}
+
+    def reset_on_duration(self, user: str):
+        # Get current and creation time
+        last_updated_at = self.user_dict[user]["last_updated_at"]
+        current_time = time.time()
+
+        # Convert duration from days to seconds
+        duration_in_seconds = self.user_dict[user]["duration"] * 24 * 60 * 60
+
+        # Check if duration has elapsed
+        if current_time - last_updated_at >= duration_in_seconds:
+            # Reset cost if duration has elapsed and update the creation time
+            self.reset_cost(user)
+            self.user_dict[user]["last_updated_at"] = current_time
+            self._save_data_thread()  # Save the data
+
+    def update_budget_all_users(self):
+        for user in self.get_users():
+            if "duration" in self.user_dict[user]:
+                self.reset_on_duration(user)
+
+    def _save_data_thread(self):
+        thread = threading.Thread(
+            target=self.save_data
+        )  # [Non-Blocking]: saves data without blocking execution
+        thread.start()
+
+    def save_data(self):
+        if self.client_type == "local":
+            import json
+
+            # save the user dict
+            with open("user_cost.json", "w") as json_file:
+                json.dump(
+                    self.user_dict, json_file, indent=4
+                )  # Indent for pretty formatting
+            return {"status": "success"}
+        elif self.client_type == "hosted":
+            url = self.api_base + "/set_budget"
+            headers = {"Content-Type": "application/json"}
+            data = {"project_name": self.project_name, "user_dict": self.user_dict}
+            response = requests.post(url, headers=headers, json=data)
+            response = response.json()
+            return response

litellm/caching.py

@@ -0,0 +1,678 @@
+# +-----------------------------------------------+
+# |                                               |
+# |           Give Feedback / Get Help            |
+# | https://github.com/BerriAI/litellm/issues/new |
+# |                                               |
+# +-----------------------------------------------+
+#
+#  Thank you users! We ❤️ you! - Krrish & Ishaan
+
+import litellm
+import time, logging
+import json, traceback, ast, hashlib
+from typing import Optional, Literal, List, Union, Any
+from openai._models import BaseModel as OpenAIObject
+
+
+def print_verbose(print_statement):
+    try:
+        if litellm.set_verbose:
+            print(print_statement)  # noqa
+    except:
+        pass
+
+
+class BaseCache:
+    def set_cache(self, key, value, **kwargs):
+        raise NotImplementedError
+
+    def get_cache(self, key, **kwargs):
+        raise NotImplementedError
+
+
+class InMemoryCache(BaseCache):
+    def __init__(self):
+        # if users don't provider one, use the default litellm cache
+        self.cache_dict = {}
+        self.ttl_dict = {}
+
+    def set_cache(self, key, value, **kwargs):
+        self.cache_dict[key] = value
+        if "ttl" in kwargs:
+            self.ttl_dict[key] = time.time() + kwargs["ttl"]
+
+    def get_cache(self, key, **kwargs):
+        if key in self.cache_dict:
+            if key in self.ttl_dict:
+                if time.time() > self.ttl_dict[key]:
+                    self.cache_dict.pop(key, None)
+                    return None
+            original_cached_response = self.cache_dict[key]
+            try:
+                cached_response = json.loads(original_cached_response)
+            except:
+                cached_response = original_cached_response
+            return cached_response
+        return None
+
+    def flush_cache(self):
+        self.cache_dict.clear()
+        self.ttl_dict.clear()
+
+
+class RedisCache(BaseCache):
+    def __init__(self, host=None, port=None, password=None, **kwargs):
+        import redis
+
+        # if users don't provider one, use the default litellm cache
+        from ._redis import get_redis_client
+
+        redis_kwargs = {}
+        if host is not None:
+            redis_kwargs["host"] = host
+        if port is not None:
+            redis_kwargs["port"] = port
+        if password is not None:
+            redis_kwargs["password"] = password
+
+        redis_kwargs.update(kwargs)
+
+        self.redis_client = get_redis_client(**redis_kwargs)
+
+    def set_cache(self, key, value, **kwargs):
+        ttl = kwargs.get("ttl", None)
+        print_verbose(f"Set Redis Cache: key: {key}\nValue {value}")
+        try:
+            self.redis_client.set(name=key, value=str(value), ex=ttl)
+        except Exception as e:
+            # NON blocking - notify users Redis is throwing an exception
+            logging.debug("LiteLLM Caching: set() - Got exception from REDIS : ", e)
+
+    def get_cache(self, key, **kwargs):
+        try:
+            print_verbose(f"Get Redis Cache: key: {key}")
+            cached_response = self.redis_client.get(key)
+            print_verbose(
+                f"Got Redis Cache: key: {key}, cached_response {cached_response}"
+            )
+            if cached_response != None:
+                # cached_response is in `b{} convert it to ModelResponse
+                cached_response = cached_response.decode(
+                    "utf-8"
+                )  # Convert bytes to string
+                try:
+                    cached_response = json.loads(
+                        cached_response
+                    )  # Convert string to dictionary
+                except:
+                    cached_response = ast.literal_eval(cached_response)
+                return cached_response
+        except Exception as e:
+            # NON blocking - notify users Redis is throwing an exception
+            traceback.print_exc()
+            logging.debug("LiteLLM Caching: get() - Got exception from REDIS: ", e)
+
+    def flush_cache(self):
+        self.redis_client.flushall()
+
+
+class S3Cache(BaseCache):
+    def __init__(
+        self,
+        s3_bucket_name,
+        s3_region_name=None,
+        s3_api_version=None,
+        s3_use_ssl=True,
+        s3_verify=None,
+        s3_endpoint_url=None,
+        s3_aws_access_key_id=None,
+        s3_aws_secret_access_key=None,
+        s3_aws_session_token=None,
+        s3_config=None,
+        **kwargs,
+    ):
+        import boto3
+
+        self.bucket_name = s3_bucket_name
+        # Create an S3 client with custom endpoint URL
+        self.s3_client = boto3.client(
+            "s3",
+            region_name=s3_region_name,
+            endpoint_url=s3_endpoint_url,
+            api_version=s3_api_version,
+            use_ssl=s3_use_ssl,
+            verify=s3_verify,
+            aws_access_key_id=s3_aws_access_key_id,
+            aws_secret_access_key=s3_aws_secret_access_key,
+            aws_session_token=s3_aws_session_token,
+            config=s3_config,
+            **kwargs,
+        )
+
+    def set_cache(self, key, value, **kwargs):
+        try:
+            print_verbose(f"LiteLLM SET Cache - S3. Key={key}. Value={value}")
+            ttl = kwargs.get("ttl", None)
+            # Convert value to JSON before storing in S3
+            serialized_value = json.dumps(value)
+            if ttl is not None:
+                cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}"
+                import datetime
+
+                # Calculate expiration time
+                expiration_time = datetime.datetime.now() + ttl
+
+                # Upload the data to S3 with the calculated expiration time
+                self.s3_client.put_object(
+                    Bucket=self.bucket_name,
+                    Key=key,
+                    Body=serialized_value,
+                    Expires=expiration_time,
+                    CacheControl=cache_control,
+                    ContentType="application/json",
+                    ContentLanguage="en",
+                    ContentDisposition=f"inline; filename=\"{key}.json\""
+                )
+            else:
+                cache_control = "immutable, max-age=31536000, s-maxage=31536000"
+                # Upload the data to S3 without specifying Expires
+                self.s3_client.put_object(
+                    Bucket=self.bucket_name,
+                    Key=key,
+                    Body=serialized_value,
+                    CacheControl=cache_control,
+                    ContentType="application/json",
+                    ContentLanguage="en",
+                    ContentDisposition=f"inline; filename=\"{key}.json\""
+                )
+        except Exception as e:
+            # NON blocking - notify users S3 is throwing an exception
+            print_verbose(f"S3 Caching: set_cache() - Got exception from S3: {e}")
+
+    def get_cache(self, key, **kwargs):
+        import boto3, botocore
+
+        try:
+            print_verbose(f"Get S3 Cache: key: {key}")
+            # Download the data from S3
+            cached_response = self.s3_client.get_object(
+                Bucket=self.bucket_name, Key=key
+            )
+
+            if cached_response != None:
+                # cached_response is in `b{} convert it to ModelResponse
+                cached_response = (
+                    cached_response["Body"].read().decode("utf-8")
+                )  # Convert bytes to string
+                try:
+                    cached_response = json.loads(
+                        cached_response
+                    )  # Convert string to dictionary
+                except Exception as e:
+                    cached_response = ast.literal_eval(cached_response)
+            if type(cached_response) is not dict:
+                cached_response = dict(cached_response)
+            print_verbose(
+                f"Got S3 Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}"
+            )
+
+            return cached_response
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "NoSuchKey":
+                print_verbose(
+                    f"S3 Cache: The specified key '{key}' does not exist in the S3 bucket."
+                )
+                return None
+
+        except Exception as e:
+            # NON blocking - notify users S3 is throwing an exception
+            traceback.print_exc()
+            print_verbose(f"S3 Caching: get_cache() - Got exception from S3: {e}")
+
+    def flush_cache(self):
+        pass
+
+
+class DualCache(BaseCache):
+    """
+    This updates both Redis and an in-memory cache simultaneously.
+    When data is updated or inserted, it is written to both the in-memory cache + Redis.
+    This ensures that even if Redis hasn't been updated yet, the in-memory cache reflects the most recent data.
+    """
+
+    def __init__(
+        self,
+        in_memory_cache: Optional[InMemoryCache] = None,
+        redis_cache: Optional[RedisCache] = None,
+    ) -> None:
+        super().__init__()
+        # If in_memory_cache is not provided, use the default InMemoryCache
+        self.in_memory_cache = in_memory_cache or InMemoryCache()
+        # If redis_cache is not provided, use the default RedisCache
+        self.redis_cache = redis_cache
+
+    def set_cache(self, key, value, local_only: bool = False, **kwargs):
+        # Update both Redis and in-memory cache
+        try:
+            print_verbose(f"set cache: key: {key}; value: {value}")
+            if self.in_memory_cache is not None:
+                self.in_memory_cache.set_cache(key, value, **kwargs)
+
+            if self.redis_cache is not None and local_only == False:
+                self.redis_cache.set_cache(key, value, **kwargs)
+        except Exception as e:
+            print_verbose(e)
+
+    def get_cache(self, key, local_only: bool = False, **kwargs):
+        # Try to fetch from in-memory cache first
+        try:
+            print_verbose(f"get cache: cache key: {key}; local_only: {local_only}")
+            result = None
+            if self.in_memory_cache is not None:
+                in_memory_result = self.in_memory_cache.get_cache(key, **kwargs)
+
+                print_verbose(f"in_memory_result: {in_memory_result}")
+                if in_memory_result is not None:
+                    result = in_memory_result
+
+            if result is None and self.redis_cache is not None and local_only == False:
+                # If not found in in-memory cache, try fetching from Redis
+                redis_result = self.redis_cache.get_cache(key, **kwargs)
+
+                if redis_result is not None:
+                    # Update in-memory cache with the value from Redis
+                    self.in_memory_cache.set_cache(key, redis_result, **kwargs)
+
+                result = redis_result
+
+            print_verbose(f"get cache: cache result: {result}")
+            return result
+        except Exception as e:
+            traceback.print_exc()
+
+    def flush_cache(self):
+        if self.in_memory_cache is not None:
+            self.in_memory_cache.flush_cache()
+        if self.redis_cache is not None:
+            self.redis_cache.flush_cache()
+
+
+#### LiteLLM.Completion / Embedding Cache ####
+class Cache:
+    def __init__(
+        self,
+        type: Optional[Literal["local", "redis", "s3"]] = "local",
+        host: Optional[str] = None,
+        port: Optional[str] = None,
+        password: Optional[str] = None,
+        supported_call_types: Optional[
+            List[Literal["completion", "acompletion", "embedding", "aembedding"]]
+        ] = ["completion", "acompletion", "embedding", "aembedding"],
+        # s3 Bucket, boto3 configuration
+        s3_bucket_name: Optional[str] = None,
+        s3_region_name: Optional[str] = None,
+        s3_api_version: Optional[str] = None,
+        s3_use_ssl: Optional[bool] = True,
+        s3_verify: Optional[Union[bool, str]] = None,
+        s3_endpoint_url: Optional[str] = None,
+        s3_aws_access_key_id: Optional[str] = None,
+        s3_aws_secret_access_key: Optional[str] = None,
+        s3_aws_session_token: Optional[str] = None,
+        s3_config: Optional[Any] = None,
+        **kwargs,
+    ):
+        """
+        Initializes the cache based on the given type.
+
+        Args:
+            type (str, optional): The type of cache to initialize. Can be "local" or "redis". Defaults to "local".
+            host (str, optional): The host address for the Redis cache. Required if type is "redis".
+            port (int, optional): The port number for the Redis cache. Required if type is "redis".
+            password (str, optional): The password for the Redis cache. Required if type is "redis".
+            supported_call_types (list, optional): List of call types to cache for. Defaults to cache == on for all call types.
+            **kwargs: Additional keyword arguments for redis.Redis() cache
+
+        Raises:
+            ValueError: If an invalid cache type is provided.
+
+        Returns:
+            None. Cache is set as a litellm param
+        """
+        if type == "redis":
+            self.cache: BaseCache = RedisCache(host, port, password, **kwargs)
+        if type == "local":
+            self.cache = InMemoryCache()
+        if type == "s3":
+            self.cache = S3Cache(
+                s3_bucket_name=s3_bucket_name,
+                s3_region_name=s3_region_name,
+                s3_api_version=s3_api_version,
+                s3_use_ssl=s3_use_ssl,
+                s3_verify=s3_verify,
+                s3_endpoint_url=s3_endpoint_url,
+                s3_aws_access_key_id=s3_aws_access_key_id,
+                s3_aws_secret_access_key=s3_aws_secret_access_key,
+                s3_aws_session_token=s3_aws_session_token,
+                s3_config=s3_config,
+                **kwargs,
+            )
+        if "cache" not in litellm.input_callback:
+            litellm.input_callback.append("cache")
+        if "cache" not in litellm.success_callback:
+            litellm.success_callback.append("cache")
+        if "cache" not in litellm._async_success_callback:
+            litellm._async_success_callback.append("cache")
+        self.supported_call_types = supported_call_types  # default to ["completion", "acompletion", "embedding", "aembedding"]
+        self.type = type
+
+    def get_cache_key(self, *args, **kwargs):
+        """
+        Get the cache key for the given arguments.
+
+        Args:
+            *args: args to litellm.completion() or embedding()
+            **kwargs: kwargs to litellm.completion() or embedding()
+
+        Returns:
+            str: The cache key generated from the arguments, or None if no cache key could be generated.
+        """
+        cache_key = ""
+        print_verbose(f"\nGetting Cache key. Kwargs: {kwargs}")
+
+        # for streaming, we use preset_cache_key. It's created in wrapper(), we do this because optional params like max_tokens, get transformed for bedrock -> max_new_tokens
+        if kwargs.get("litellm_params", {}).get("preset_cache_key", None) is not None:
+            print_verbose(f"\nReturning preset cache key: {cache_key}")
+            return kwargs.get("litellm_params", {}).get("preset_cache_key", None)
+
+        # sort kwargs by keys, since model: [gpt-4, temperature: 0.2, max_tokens: 200] == [temperature: 0.2, max_tokens: 200, model: gpt-4]
+        completion_kwargs = [
+            "model",
+            "messages",
+            "temperature",
+            "top_p",
+            "n",
+            "stop",
+            "max_tokens",
+            "presence_penalty",
+            "frequency_penalty",
+            "logit_bias",
+            "user",
+            "response_format",
+            "seed",
+            "tools",
+            "tool_choice",
+        ]
+        embedding_only_kwargs = [
+            "input",
+            "encoding_format",
+        ]  # embedding kwargs = model, input, user, encoding_format. Model, user are checked in completion_kwargs
+
+        # combined_kwargs - NEEDS to be ordered across get_cache_key(). Do not use a set()
+        combined_kwargs = completion_kwargs + embedding_only_kwargs
+        for param in combined_kwargs:
+            # ignore litellm params here
+            if param in kwargs:
+                # check if param == model and model_group is passed in, then override model with model_group
+                if param == "model":
+                    model_group = None
+                    caching_group = None
+                    metadata = kwargs.get("metadata", None)
+                    litellm_params = kwargs.get("litellm_params", {})
+                    if metadata is not None:
+                        model_group = metadata.get("model_group")
+                        model_group = metadata.get("model_group", None)
+                        caching_groups = metadata.get("caching_groups", None)
+                        if caching_groups:
+                            for group in caching_groups:
+                                if model_group in group:
+                                    caching_group = group
+                                    break
+                    if litellm_params is not None:
+                        metadata = litellm_params.get("metadata", None)
+                        if metadata is not None:
+                            model_group = metadata.get("model_group", None)
+                            caching_groups = metadata.get("caching_groups", None)
+                            if caching_groups:
+                                for group in caching_groups:
+                                    if model_group in group:
+                                        caching_group = group
+                                        break
+                    param_value = (
+                        caching_group or model_group or kwargs[param]
+                    )  # use caching_group, if set then model_group if it exists, else use kwargs["model"]
+                else:
+                    if kwargs[param] is None:
+                        continue  # ignore None params
+                    param_value = kwargs[param]
+                cache_key += f"{str(param)}: {str(param_value)}"
+        print_verbose(f"\nCreated cache key: {cache_key}")
+        # Use hashlib to create a sha256 hash of the cache key
+        hash_object = hashlib.sha256(cache_key.encode())
+        # Hexadecimal representation of the hash
+        hash_hex = hash_object.hexdigest()
+        print_verbose(f"Hashed cache key (SHA-256): {hash_hex}")
+        return hash_hex
+
+    def generate_streaming_content(self, content):
+        chunk_size = 5  # Adjust the chunk size as needed
+        for i in range(0, len(content), chunk_size):
+            yield {
+                "choices": [
+                    {
+                        "delta": {
+                            "role": "assistant",
+                            "content": content[i : i + chunk_size],
+                        }
+                    }
+                ]
+            }
+            time.sleep(0.02)
+
+    def get_cache(self, *args, **kwargs):
+        """
+        Retrieves the cached result for the given arguments.
+
+        Args:
+            *args: args to litellm.completion() or embedding()
+            **kwargs: kwargs to litellm.completion() or embedding()
+
+        Returns:
+            The cached result if it exists, otherwise None.
+        """
+        try:  # never block execution
+            if "cache_key" in kwargs:
+                cache_key = kwargs["cache_key"]
+            else:
+                cache_key = self.get_cache_key(*args, **kwargs)
+            if cache_key is not None:
+                cache_control_args = kwargs.get("cache", {})
+                max_age = cache_control_args.get(
+                    "s-max-age", cache_control_args.get("s-maxage", float("inf"))
+                )
+                cached_result = self.cache.get_cache(cache_key)
+                # Check if a timestamp was stored with the cached response
+                if (
+                    cached_result is not None
+                    and isinstance(cached_result, dict)
+                    and "timestamp" in cached_result
+                    and max_age is not None
+                ):
+                    timestamp = cached_result["timestamp"]
+                    current_time = time.time()
+
+                    # Calculate age of the cached response
+                    response_age = current_time - timestamp
+
+                    # Check if the cached response is older than the max-age
+                    if response_age > max_age:
+                        print_verbose(
+                            f"Cached response for key {cache_key} is too old. Max-age: {max_age}s, Age: {response_age}s"
+                        )
+                        return None  # Cached response is too old
+
+                    # If the response is fresh, or there's no max-age requirement, return the cached response
+                    # cached_response is in `b{} convert it to ModelResponse
+                    cached_response = cached_result.get("response")
+                    try:
+                        if isinstance(cached_response, dict):
+                            pass
+                        else:
+                            cached_response = json.loads(
+                                cached_response
+                            )  # Convert string to dictionary
+                    except:
+                        cached_response = ast.literal_eval(cached_response)
+                    return cached_response
+                return cached_result
+        except Exception as e:
+            print_verbose(f"An exception occurred: {traceback.format_exc()}")
+            return None
+
+    def add_cache(self, result, *args, **kwargs):
+        """
+        Adds a result to the cache.
+
+        Args:
+            *args: args to litellm.completion() or embedding()
+            **kwargs: kwargs to litellm.completion() or embedding()
+
+        Returns:
+            None
+        """
+        try:
+            if "cache_key" in kwargs:
+                cache_key = kwargs["cache_key"]
+            else:
+                cache_key = self.get_cache_key(*args, **kwargs)
+            if cache_key is not None:
+                if isinstance(result, OpenAIObject):
+                    result = result.model_dump_json()
+
+                ## Get Cache-Controls ##
+                if kwargs.get("cache", None) is not None and isinstance(
+                    kwargs.get("cache"), dict
+                ):
+                    for k, v in kwargs.get("cache").items():
+                        if k == "ttl":
+                            kwargs["ttl"] = v
+                cached_data = {"timestamp": time.time(), "response": result}
+                self.cache.set_cache(cache_key, cached_data, **kwargs)
+        except Exception as e:
+            print_verbose(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
+            traceback.print_exc()
+            pass
+
+    async def _async_add_cache(self, result, *args, **kwargs):
+        self.add_cache(result, *args, **kwargs)
+
+
+def enable_cache(
+    type: Optional[Literal["local", "redis", "s3"]] = "local",
+    host: Optional[str] = None,
+    port: Optional[str] = None,
+    password: Optional[str] = None,
+    supported_call_types: Optional[
+        List[Literal["completion", "acompletion", "embedding", "aembedding"]]
+    ] = ["completion", "acompletion", "embedding", "aembedding"],
+    **kwargs,
+):
+    """
+    Enable cache with the specified configuration.
+
+    Args:
+        type (Optional[Literal["local", "redis"]]): The type of cache to enable. Defaults to "local".
+        host (Optional[str]): The host address of the cache server. Defaults to None.
+        port (Optional[str]): The port number of the cache server. Defaults to None.
+        password (Optional[str]): The password for the cache server. Defaults to None.
+        supported_call_types (Optional[List[Literal["completion", "acompletion", "embedding", "aembedding"]]]):
+            The supported call types for the cache. Defaults to ["completion", "acompletion", "embedding", "aembedding"].
+        **kwargs: Additional keyword arguments.
+
+    Returns:
+        None
+
+    Raises:
+        None
+    """
+    print_verbose("LiteLLM: Enabling Cache")
+    if "cache" not in litellm.input_callback:
+        litellm.input_callback.append("cache")
+    if "cache" not in litellm.success_callback:
+        litellm.success_callback.append("cache")
+    if "cache" not in litellm._async_success_callback:
+        litellm._async_success_callback.append("cache")
+
+    if litellm.cache == None:
+        litellm.cache = Cache(
+            type=type,
+            host=host,
+            port=port,
+            password=password,
+            supported_call_types=supported_call_types,
+            **kwargs,
+        )
+    print_verbose(f"LiteLLM: Cache enabled, litellm.cache={litellm.cache}")
+    print_verbose(f"LiteLLM Cache: {vars(litellm.cache)}")
+
+
+def update_cache(
+    type: Optional[Literal["local", "redis"]] = "local",
+    host: Optional[str] = None,
+    port: Optional[str] = None,
+    password: Optional[str] = None,
+    supported_call_types: Optional[
+        List[Literal["completion", "acompletion", "embedding", "aembedding"]]
+    ] = ["completion", "acompletion", "embedding", "aembedding"],
+    **kwargs,
+):
+    """
+    Update the cache for LiteLLM.
+
+    Args:
+        type (Optional[Literal["local", "redis"]]): The type of cache. Defaults to "local".
+        host (Optional[str]): The host of the cache. Defaults to None.
+        port (Optional[str]): The port of the cache. Defaults to None.
+        password (Optional[str]): The password for the cache. Defaults to None.
+        supported_call_types (Optional[List[Literal["completion", "acompletion", "embedding", "aembedding"]]]):
+            The supported call types for the cache. Defaults to ["completion", "acompletion", "embedding", "aembedding"].
+        **kwargs: Additional keyword arguments for the cache.
+
+    Returns:
+        None
+
+    """
+    print_verbose("LiteLLM: Updating Cache")
+    litellm.cache = Cache(
+        type=type,
+        host=host,
+        port=port,
+        password=password,
+        supported_call_types=supported_call_types,
+        **kwargs,
+    )
+    print_verbose(f"LiteLLM: Cache Updated, litellm.cache={litellm.cache}")
+    print_verbose(f"LiteLLM Cache: {vars(litellm.cache)}")
+
+
+def disable_cache():
+    """
+    Disable the cache used by LiteLLM.
+
+    This function disables the cache used by the LiteLLM module. It removes the cache-related callbacks from the input_callback, success_callback, and _async_success_callback lists. It also sets the litellm.cache attribute to None.
+
+    Parameters:
+    None
+
+    Returns:
+    None
+    """
+    from contextlib import suppress
+
+    print_verbose("LiteLLM: Disabling Cache")
+    with suppress(ValueError):
+        litellm.input_callback.remove("cache")
+        litellm.success_callback.remove("cache")
+        litellm._async_success_callback.remove("cache")
+
+    litellm.cache = None
+    print_verbose(f"LiteLLM: Cache disabled, litellm.cache={litellm.cache}")

litellm/cost.json

@@ -0,0 +1,5 @@
+{
+    "gpt-3.5-turbo-0613": 0.00015000000000000001,
+    "claude-2": 0.00016454,
+    "gpt-4-0613": 0.015408
+}

litellm/deprecated_litellm_server/.env.template

@@ -0,0 +1,43 @@
+# # set AUTH STRATEGY FOR LLM APIs - Defaults to using Environment Variables
+# AUTH_STRATEGY = "ENV" # ENV or DYNAMIC, ENV always reads from environment variables, DYNAMIC reads request headers to set LLM api keys
+
+# OPENAI_API_KEY = ""
+
+# HUGGINGFACE_API_KEY="" 
+
+# TOGETHERAI_API_KEY=""
+
+# REPLICATE_API_KEY="" 
+
+# ## bedrock / sagemaker
+# AWS_ACCESS_KEY_ID = "" 
+# AWS_SECRET_ACCESS_KEY = ""
+
+# AZURE_API_KEY = ""
+# AZURE_API_BASE = ""
+# AZURE_API_VERSION = ""
+
+# ANTHROPIC_API_KEY = ""
+
+# COHERE_API_KEY = ""
+
+# ## CONFIG FILE ## 
+# # CONFIG_FILE_PATH = ""  # uncomment to point to config file  
+
+# ## LOGGING ## 
+
+# SET_VERBOSE = "False" # set to 'True' to see detailed input/output logs
+
+# ### LANGFUSE
+# LANGFUSE_PUBLIC_KEY = ""
+# LANGFUSE_SECRET_KEY = ""
+# # Optional, defaults to https://cloud.langfuse.com
+# LANGFUSE_HOST = "" # optional
+
+
+# ## CACHING ## 
+
+# ### REDIS
+# REDIS_HOST = "" 
+# REDIS_PORT = "" 
+# REDIS_PASSWORD = "" 

litellm/deprecated_litellm_server/Dockerfile

@@ -0,0 +1,10 @@
+# FROM python:3.10
+
+# ENV LITELLM_CONFIG_PATH="/litellm.secrets.toml"
+# COPY . /app
+# WORKDIR /app
+# RUN pip install -r requirements.txt
+
+# EXPOSE $PORT 
+
+# CMD exec uvicorn main:app --host 0.0.0.0 --port $PORT --workers 10 

litellm/deprecated_litellm_server/README.md

@@ -0,0 +1,3 @@
+# litellm-server [experimental]
+
+Deprecated. See litellm/proxy

litellm/deprecated_litellm_server/__init__.py

@@ -0,0 +1,2 @@
+# from .main import *
+# from .server_utils import *

litellm/deprecated_litellm_server/main.py

@@ -0,0 +1,193 @@
+# import os, traceback
+# from fastapi import FastAPI, Request, HTTPException
+# from fastapi.routing import APIRouter
+# from fastapi.responses import StreamingResponse, FileResponse
+# from fastapi.middleware.cors import CORSMiddleware
+# import json, sys
+# from typing import Optional
+# sys.path.insert(
+#     0, os.path.abspath("../")
+# )  # Adds the parent directory to the system path - for litellm local dev
+# import litellm
+
+# try:
+#     from litellm.deprecated_litellm_server.server_utils import set_callbacks, load_router_config, print_verbose
+# except ImportError:
+#     from litellm.deprecated_litellm_server.server_utils import set_callbacks, load_router_config, print_verbose
+# import dotenv
+# dotenv.load_dotenv() # load env variables
+
+# app = FastAPI(docs_url="/", title="LiteLLM API")
+# router = APIRouter()
+# origins = ["*"]
+
+# app.add_middleware(
+#     CORSMiddleware,
+#     allow_origins=origins,
+#     allow_credentials=True,
+#     allow_methods=["*"],
+#     allow_headers=["*"],
+# )
+# #### GLOBAL VARIABLES ####
+# llm_router: Optional[litellm.Router] = None
+# llm_model_list: Optional[list] = None
+# server_settings: Optional[dict] = None
+
+# set_callbacks() # sets litellm callbacks for logging if they exist in the environment
+
+# if "CONFIG_FILE_PATH" in os.environ:
+#     llm_router, llm_model_list, server_settings = load_router_config(router=llm_router, config_file_path=os.getenv("CONFIG_FILE_PATH"))
+# else:
+#     llm_router, llm_model_list, server_settings = load_router_config(router=llm_router)
+# #### API ENDPOINTS ####
+# @router.get("/v1/models")
+# @router.get("/models")  # if project requires model list
+# def model_list():
+#     all_models = litellm.utils.get_valid_models()
+#     if llm_model_list:
+#         all_models += llm_model_list
+#     return dict(
+#         data=[
+#             {
+#                 "id": model,
+#                 "object": "model",
+#                 "created": 1677610602,
+#                 "owned_by": "openai",
+#             }
+#             for model in all_models
+#         ],
+#         object="list",
+#     )
+# # for streaming
+# def data_generator(response):
+
+#     for chunk in response:
+
+#         yield f"data: {json.dumps(chunk)}\n\n"
+
+# @router.post("/v1/completions")
+# @router.post("/completions")
+# async def completion(request: Request):
+#     data = await request.json()
+#     response = litellm.completion(
+#         **data
+#     )
+#     if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses
+#             return StreamingResponse(data_generator(response), media_type='text/event-stream')
+#     return response
+
+# @router.post("/v1/embeddings")
+# @router.post("/embeddings")
+# async def embedding(request: Request):
+#     try:
+#         data = await request.json()
+#         # default to always using the "ENV" variables, only if AUTH_STRATEGY==DYNAMIC then reads headers
+#         if os.getenv("AUTH_STRATEGY", None) == "DYNAMIC" and "authorization" in request.headers: # if users pass LLM api keys as part of header
+#             api_key = request.headers.get("authorization")
+#             api_key = api_key.replace("Bearer", "").strip() # type: ignore
+#             if len(api_key.strip()) > 0:
+#                 api_key = api_key
+#                 data["api_key"] = api_key
+#         response = litellm.embedding(
+#             **data
+#         )
+#         return response
+#     except Exception as e:
+#         error_traceback = traceback.format_exc()
+#         error_msg = f"{str(e)}\n\n{error_traceback}"
+#         return {"error": error_msg}
+
+# @router.post("/v1/chat/completions")
+# @router.post("/chat/completions")
+# @router.post("/openai/deployments/{model:path}/chat/completions") # azure compatible endpoint
+# async def chat_completion(request: Request, model: Optional[str] = None):
+#     global llm_model_list, server_settings
+#     try:
+#         data = await request.json()
+#         server_model = server_settings.get("completion_model", None) if server_settings else None
+#         data["model"] = server_model or model or data["model"]
+#         ## CHECK KEYS ##
+#         # default to always using the "ENV" variables, only if AUTH_STRATEGY==DYNAMIC then reads headers
+#         # env_validation = litellm.validate_environment(model=data["model"])
+#         # if (env_validation['keys_in_environment'] is False or os.getenv("AUTH_STRATEGY", None) == "DYNAMIC") and ("authorization" in request.headers or "api-key" in request.headers): # if users pass LLM api keys as part of header
+#         #     if "authorization" in request.headers:
+#         #         api_key = request.headers.get("authorization")
+#         #     elif "api-key" in request.headers:
+#         #         api_key = request.headers.get("api-key")
+#         #     print(f"api_key in headers: {api_key}")
+#         #     if " " in api_key:
+#         #         api_key = api_key.split(" ")[1]
+#         #     print(f"api_key split: {api_key}")
+#         #     if len(api_key) > 0:
+#         #         api_key = api_key
+#         #         data["api_key"] = api_key
+#         #         print(f"api_key in data: {api_key}")
+#         ## CHECK CONFIG ##
+#         if llm_model_list and data["model"] in [m["model_name"] for m in llm_model_list]:
+#             for m in llm_model_list:
+#                 if data["model"] == m["model_name"]:
+#                     for key, value in m["litellm_params"].items():
+#                         data[key] = value
+#                     break
+#         response = litellm.completion(
+#             **data
+#         )
+#         if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses
+#                 return StreamingResponse(data_generator(response), media_type='text/event-stream')
+#         return response
+#     except Exception as e:
+#         error_traceback = traceback.format_exc()
+
+#         error_msg = f"{str(e)}\n\n{error_traceback}"
+#         # return {"error": error_msg}
+#         raise HTTPException(status_code=500, detail=error_msg)
+
+# @router.post("/router/completions")
+# async def router_completion(request: Request):
+#     global llm_router
+#     try:
+#         data = await request.json()
+#         if "model_list" in data:
+#             llm_router = litellm.Router(model_list=data.pop("model_list"))
+#         if llm_router is None:
+#             raise Exception("Save model list via config.yaml. Eg.: ` docker build -t myapp --build-arg CONFIG_FILE=myconfig.yaml .` or pass it in as model_list=[..] as part of the request body")
+
+#         # openai.ChatCompletion.create replacement
+#         response = await llm_router.acompletion(model="gpt-3.5-turbo",
+#                         messages=[{"role": "user", "content": "Hey, how's it going?"}])
+
+#         if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses
+#                 return StreamingResponse(data_generator(response), media_type='text/event-stream')
+#         return response
+#     except Exception as e:
+#         error_traceback = traceback.format_exc()
+#         error_msg = f"{str(e)}\n\n{error_traceback}"
+#         return {"error": error_msg}
+
+# @router.post("/router/embedding")
+# async def router_embedding(request: Request):
+#     global llm_router
+#     try:
+#         data = await request.json()
+#         if "model_list" in data:
+#             llm_router = litellm.Router(model_list=data.pop("model_list"))
+#         if llm_router is None:
+#             raise Exception("Save model list via config.yaml. Eg.: ` docker build -t myapp --build-arg CONFIG_FILE=myconfig.yaml .` or pass it in as model_list=[..] as part of the request body")
+
+#         response = await llm_router.aembedding(model="gpt-3.5-turbo",  # type: ignore
+#                         messages=[{"role": "user", "content": "Hey, how's it going?"}])
+
+#         if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses
+#                 return StreamingResponse(data_generator(response), media_type='text/event-stream')
+#         return response
+#     except Exception as e:
+#         error_traceback = traceback.format_exc()
+#         error_msg = f"{str(e)}\n\n{error_traceback}"
+#         return {"error": error_msg}
+
+# @router.get("/")
+# async def home(request: Request):
+#     return "LiteLLM: RUNNING"
+
+
+# app.include_router(router)

litellm/deprecated_litellm_server/requirements.txt

@@ -0,0 +1,7 @@
+# openai
+# fastapi
+# uvicorn
+# boto3
+# litellm
+# python-dotenv
+# redis

litellm/deprecated_litellm_server/server_utils.py

@@ -0,0 +1,85 @@
+# import os, litellm
+# import pkg_resources
+# import dotenv
+# dotenv.load_dotenv() # load env variables
+
+# def print_verbose(print_statement):
+#     pass
+
+# def get_package_version(package_name):
+#     try:
+#         package = pkg_resources.get_distribution(package_name)
+#         return package.version
+#     except pkg_resources.DistributionNotFound:
+#         return None
+
+# # Usage example
+# package_name = "litellm"
+# version = get_package_version(package_name)
+# if version:
+#     print_verbose(f"The version of {package_name} is {version}")
+# else:
+#     print_verbose(f"{package_name} is not installed")
+# import yaml
+# import dotenv
+# from typing import Optional
+# dotenv.load_dotenv() # load env variables
+
+# def set_callbacks():
+#     ## LOGGING
+#     if len(os.getenv("SET_VERBOSE", "")) > 0:
+#         if os.getenv("SET_VERBOSE") == "True":
+#             litellm.set_verbose = True
+#             print_verbose("\033[92mLiteLLM: Switched on verbose logging\033[0m")
+#         else:
+#             litellm.set_verbose = False
+
+#     ### LANGFUSE
+#     if (len(os.getenv("LANGFUSE_PUBLIC_KEY", "")) > 0 and len(os.getenv("LANGFUSE_SECRET_KEY", ""))) > 0 or len(os.getenv("LANGFUSE_HOST", "")) > 0:
+#         litellm.success_callback = ["langfuse"]
+#         print_verbose("\033[92mLiteLLM: Switched on Langfuse feature\033[0m")
+
+#     ## CACHING
+#     ### REDIS
+#     # if len(os.getenv("REDIS_HOST", "")) >  0 and len(os.getenv("REDIS_PORT", "")) > 0 and len(os.getenv("REDIS_PASSWORD", "")) > 0:
+#     #     print(f"redis host: {os.getenv('REDIS_HOST')}; redis port: {os.getenv('REDIS_PORT')}; password: {os.getenv('REDIS_PASSWORD')}")
+#     #     from litellm.caching import Cache
+#     #     litellm.cache = Cache(type="redis", host=os.getenv("REDIS_HOST"), port=os.getenv("REDIS_PORT"), password=os.getenv("REDIS_PASSWORD"))
+#     #     print("\033[92mLiteLLM: Switched on Redis caching\033[0m")
+
+
+# def load_router_config(router: Optional[litellm.Router], config_file_path: Optional[str]='/app/config.yaml'):
+#     config = {}
+#     server_settings  = {}
+#     try:
+#         if os.path.exists(config_file_path): # type: ignore
+#             with open(config_file_path, 'r') as file: # type: ignore
+#                 config = yaml.safe_load(file)
+#         else:
+#             pass
+#     except:
+#         pass
+
+#     ## SERVER SETTINGS (e.g. default completion model = 'ollama/mistral')
+#     server_settings = config.get("server_settings", None)
+#     if server_settings:
+#         server_settings = server_settings
+
+#     ## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
+#     litellm_settings = config.get('litellm_settings', None)
+#     if litellm_settings:
+#         for key, value in litellm_settings.items():
+#             setattr(litellm, key, value)
+
+#     ## MODEL LIST
+#     model_list = config.get('model_list', None)
+#     if model_list:
+#         router = litellm.Router(model_list=model_list)
+
+#     ## ENVIRONMENT VARIABLES
+#     environment_variables = config.get('environment_variables', None)
+#     if environment_variables:
+#         for key, value in environment_variables.items():
+#             os.environ[key] = value
+
+#     return router, model_list, server_settings

litellm/exceptions.py

@@ -0,0 +1,200 @@
+# +-----------------------------------------------+
+# |                                               |
+# |           Give Feedback / Get Help            |
+# | https://github.com/BerriAI/litellm/issues/new |
+# |                                               |
+# +-----------------------------------------------+
+#
+#  Thank you users! We ❤️ you! - Krrish & Ishaan
+
+## LiteLLM versions of the OpenAI Exception Types
+
+from openai import (
+    AuthenticationError,
+    BadRequestError,
+    NotFoundError,
+    RateLimitError,
+    APIStatusError,
+    OpenAIError,
+    APIError,
+    APITimeoutError,
+    APIConnectionError,
+    APIResponseValidationError,
+    UnprocessableEntityError,
+)
+import httpx
+
+
+class AuthenticationError(AuthenticationError):  # type: ignore
+    def __init__(self, message, llm_provider, model, response: httpx.Response):
+        self.status_code = 401
+        self.message = message
+        self.llm_provider = llm_provider
+        self.model = model
+        super().__init__(
+            self.message, response=response, body=None
+        )  # Call the base class constructor with the parameters it needs
+
+
+# raise when invalid models passed, example gpt-8
+class NotFoundError(NotFoundError):  # type: ignore
+    def __init__(self, message, model, llm_provider, response: httpx.Response):
+        self.status_code = 404
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        super().__init__(
+            self.message, response=response, body=None
+        )  # Call the base class constructor with the parameters it needs
+
+
+class BadRequestError(BadRequestError):  # type: ignore
+    def __init__(self, message, model, llm_provider, response: httpx.Response):
+        self.status_code = 400
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        super().__init__(
+            self.message, response=response, body=None
+        )  # Call the base class constructor with the parameters it needs
+
+
+class UnprocessableEntityError(UnprocessableEntityError):  # type: ignore
+    def __init__(self, message, model, llm_provider, response: httpx.Response):
+        self.status_code = 422
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        super().__init__(
+            self.message, response=response, body=None
+        )  # Call the base class constructor with the parameters it needs
+
+
+class Timeout(APITimeoutError):  # type: ignore
+    def __init__(self, message, model, llm_provider):
+        self.status_code = 408
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        request = httpx.Request(method="POST", url="https://api.openai.com/v1")
+        super().__init__(
+            request=request
+        )  # Call the base class constructor with the parameters it needs
+
+
+class RateLimitError(RateLimitError):  # type: ignore
+    def __init__(self, message, llm_provider, model, response: httpx.Response):
+        self.status_code = 429
+        self.message = message
+        self.llm_provider = llm_provider
+        self.modle = model
+        super().__init__(
+            self.message, response=response, body=None
+        )  # Call the base class constructor with the parameters it needs
+
+
+# sub class of rate limit error - meant to give more granularity for error handling context window exceeded errors
+class ContextWindowExceededError(BadRequestError):  # type: ignore
+    def __init__(self, message, model, llm_provider, response: httpx.Response):
+        self.status_code = 400
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        super().__init__(
+            message=self.message,
+            model=self.model,  # type: ignore
+            llm_provider=self.llm_provider,  # type: ignore
+            response=response,
+        )  # Call the base class constructor with the parameters it needs
+
+
+class ContentPolicyViolationError(BadRequestError):  # type: ignore
+    #  Error code: 400 - {'error': {'code': 'content_policy_violation', 'message': 'Your request was rejected as a result of our safety system. Image descriptions generated from your prompt may contain text that is not allowed by our safety system. If you believe this was done in error, your request may succeed if retried, or by adjusting your prompt.', 'param': None, 'type': 'invalid_request_error'}}
+    def __init__(self, message, model, llm_provider, response: httpx.Response):
+        self.status_code = 400
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        super().__init__(
+            message=self.message,
+            model=self.model,  # type: ignore
+            llm_provider=self.llm_provider,  # type: ignore
+            response=response,
+        )  # Call the base class constructor with the parameters it needs
+
+
+class ServiceUnavailableError(APIStatusError):  # type: ignore
+    def __init__(self, message, llm_provider, model, response: httpx.Response):
+        self.status_code = 503
+        self.message = message
+        self.llm_provider = llm_provider
+        self.model = model
+        super().__init__(
+            self.message, response=response, body=None
+        )  # Call the base class constructor with the parameters it needs
+
+
+# raise this when the API returns an invalid response object - https://github.com/openai/openai-python/blob/1be14ee34a0f8e42d3f9aa5451aa4cb161f1781f/openai/api_requestor.py#L401
+class APIError(APIError):  # type: ignore
+    def __init__(
+        self, status_code, message, llm_provider, model, request: httpx.Request
+    ):
+        self.status_code = status_code
+        self.message = message
+        self.llm_provider = llm_provider
+        self.model = model
+        super().__init__(self.message, request=request, body=None)  # type: ignore
+
+
+# raised if an invalid request (not get, delete, put, post) is made
+class APIConnectionError(APIConnectionError):  # type: ignore
+    def __init__(self, message, llm_provider, model, request: httpx.Request):
+        self.message = message
+        self.llm_provider = llm_provider
+        self.model = model
+        self.status_code = 500
+        super().__init__(message=self.message, request=request)
+
+
+# raised if an invalid request (not get, delete, put, post) is made
+class APIResponseValidationError(APIResponseValidationError):  # type: ignore
+    def __init__(self, message, llm_provider, model):
+        self.message = message
+        self.llm_provider = llm_provider
+        self.model = model
+        request = httpx.Request(method="POST", url="https://api.openai.com/v1")
+        response = httpx.Response(status_code=500, request=request)
+        super().__init__(response=response, body=None, message=message)
+
+
+class OpenAIError(OpenAIError):  # type: ignore
+    def __init__(self, original_exception):
+        self.status_code = original_exception.http_status
+        super().__init__(
+            http_body=original_exception.http_body,
+            http_status=original_exception.http_status,
+            json_body=original_exception.json_body,
+            headers=original_exception.headers,
+            code=original_exception.code,
+        )
+        self.llm_provider = "openai"
+
+
+class BudgetExceededError(Exception):
+    def __init__(self, current_cost, max_budget):
+        self.current_cost = current_cost
+        self.max_budget = max_budget
+        message = f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}"
+        super().__init__(message)
+
+
+## DEPRECATED ##
+class InvalidRequestError(BadRequestError):  # type: ignore
+    def __init__(self, message, model, llm_provider):
+        self.status_code = 400
+        self.message = message
+        self.model = model
+        self.llm_provider = llm_provider
+        super().__init__(
+            self.message, f"{self.model}"
+        )  # Call the base class constructor with the parameters it needs

litellm/integrations/__init__.py

@@ -0,0 +1 @@
+from . import *

litellm/integrations/aispend.py

@@ -0,0 +1,177 @@
+#### What this does ####
+#    On success + failure, log events to aispend.io
+import dotenv, os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+import datetime
+
+model_cost = {
+    "gpt-3.5-turbo": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },
+    "gpt-35-turbo": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },  # azure model name
+    "gpt-3.5-turbo-0613": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },
+    "gpt-3.5-turbo-0301": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },
+    "gpt-3.5-turbo-16k": {
+        "max_tokens": 16000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+    },
+    "gpt-35-turbo-16k": {
+        "max_tokens": 16000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+    },  # azure model name
+    "gpt-3.5-turbo-16k-0613": {
+        "max_tokens": 16000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+    },
+    "gpt-4": {
+        "max_tokens": 8000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.00006,
+    },
+    "gpt-4-0613": {
+        "max_tokens": 8000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.00006,
+    },
+    "gpt-4-32k": {
+        "max_tokens": 8000,
+        "input_cost_per_token": 0.00006,
+        "output_cost_per_token": 0.00012,
+    },
+    "claude-instant-1": {
+        "max_tokens": 100000,
+        "input_cost_per_token": 0.00000163,
+        "output_cost_per_token": 0.00000551,
+    },
+    "claude-2": {
+        "max_tokens": 100000,
+        "input_cost_per_token": 0.00001102,
+        "output_cost_per_token": 0.00003268,
+    },
+    "text-bison-001": {
+        "max_tokens": 8192,
+        "input_cost_per_token": 0.000004,
+        "output_cost_per_token": 0.000004,
+    },
+    "chat-bison-001": {
+        "max_tokens": 4096,
+        "input_cost_per_token": 0.000002,
+        "output_cost_per_token": 0.000002,
+    },
+    "command-nightly": {
+        "max_tokens": 4096,
+        "input_cost_per_token": 0.000015,
+        "output_cost_per_token": 0.000015,
+    },
+}
+
+
+class AISpendLogger:
+    # Class variables or attributes
+    def __init__(self):
+        # Instance variables
+        self.account_id = os.getenv("AISPEND_ACCOUNT_ID")
+        self.api_key = os.getenv("AISPEND_API_KEY")
+
+    def price_calculator(self, model, response_obj, start_time, end_time):
+        # try and find if the model is in the model_cost map
+        # else default to the average of the costs
+        prompt_tokens_cost_usd_dollar = 0
+        completion_tokens_cost_usd_dollar = 0
+        if model in model_cost:
+            prompt_tokens_cost_usd_dollar = (
+                model_cost[model]["input_cost_per_token"]
+                * response_obj["usage"]["prompt_tokens"]
+            )
+            completion_tokens_cost_usd_dollar = (
+                model_cost[model]["output_cost_per_token"]
+                * response_obj["usage"]["completion_tokens"]
+            )
+        elif "replicate" in model:
+            # replicate models are charged based on time
+            # llama 2 runs on an nvidia a100 which costs $0.0032 per second - https://replicate.com/replicate/llama-2-70b-chat
+            model_run_time = end_time - start_time  # assuming time in seconds
+            cost_usd_dollar = model_run_time * 0.0032
+            prompt_tokens_cost_usd_dollar = cost_usd_dollar / 2
+            completion_tokens_cost_usd_dollar = cost_usd_dollar / 2
+        else:
+            # calculate average input cost
+            input_cost_sum = 0
+            output_cost_sum = 0
+            for model in model_cost:
+                input_cost_sum += model_cost[model]["input_cost_per_token"]
+                output_cost_sum += model_cost[model]["output_cost_per_token"]
+            avg_input_cost = input_cost_sum / len(model_cost.keys())
+            avg_output_cost = output_cost_sum / len(model_cost.keys())
+            prompt_tokens_cost_usd_dollar = (
+                model_cost[model]["input_cost_per_token"]
+                * response_obj["usage"]["prompt_tokens"]
+            )
+            completion_tokens_cost_usd_dollar = (
+                model_cost[model]["output_cost_per_token"]
+                * response_obj["usage"]["completion_tokens"]
+            )
+        return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
+
+    def log_event(self, model, response_obj, start_time, end_time, print_verbose):
+        # Method definition
+        try:
+            print_verbose(
+                f"AISpend Logging - Enters logging function for model {model}"
+            )
+
+            url = f"https://aispend.io/api/v1/accounts/{self.account_id}/data"
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+
+            response_timestamp = datetime.datetime.fromtimestamp(
+                int(response_obj["created"])
+            ).strftime("%Y-%m-%d")
+
+            (
+                prompt_tokens_cost_usd_dollar,
+                completion_tokens_cost_usd_dollar,
+            ) = self.price_calculator(model, response_obj, start_time, end_time)
+            prompt_tokens_cost_usd_cent = prompt_tokens_cost_usd_dollar * 100
+            completion_tokens_cost_usd_cent = completion_tokens_cost_usd_dollar * 100
+            data = [
+                {
+                    "requests": 1,
+                    "requests_context": 1,
+                    "context_tokens": response_obj["usage"]["prompt_tokens"],
+                    "requests_generated": 1,
+                    "generated_tokens": response_obj["usage"]["completion_tokens"],
+                    "recorded_date": response_timestamp,
+                    "model_id": response_obj["model"],
+                    "generated_tokens_cost_usd_cent": prompt_tokens_cost_usd_cent,
+                    "context_tokens_cost_usd_cent": completion_tokens_cost_usd_cent,
+                }
+            ]
+
+            print_verbose(f"AISpend Logging - final data object: {data}")
+        except:
+            # traceback.print_exc()
+            print_verbose(f"AISpend Logging Error - {traceback.format_exc()}")
+            pass

litellm/integrations/berrispend.py

@@ -0,0 +1,184 @@
+#### What this does ####
+#    On success + failure, log events to aispend.io
+import dotenv, os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+import datetime
+
+model_cost = {
+    "gpt-3.5-turbo": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },
+    "gpt-35-turbo": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },  # azure model name
+    "gpt-3.5-turbo-0613": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },
+    "gpt-3.5-turbo-0301": {
+        "max_tokens": 4000,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+    },
+    "gpt-3.5-turbo-16k": {
+        "max_tokens": 16000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+    },
+    "gpt-35-turbo-16k": {
+        "max_tokens": 16000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+    },  # azure model name
+    "gpt-3.5-turbo-16k-0613": {
+        "max_tokens": 16000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+    },
+    "gpt-4": {
+        "max_tokens": 8000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.00006,
+    },
+    "gpt-4-0613": {
+        "max_tokens": 8000,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.00006,
+    },
+    "gpt-4-32k": {
+        "max_tokens": 8000,
+        "input_cost_per_token": 0.00006,
+        "output_cost_per_token": 0.00012,
+    },
+    "claude-instant-1": {
+        "max_tokens": 100000,
+        "input_cost_per_token": 0.00000163,
+        "output_cost_per_token": 0.00000551,
+    },
+    "claude-2": {
+        "max_tokens": 100000,
+        "input_cost_per_token": 0.00001102,
+        "output_cost_per_token": 0.00003268,
+    },
+    "text-bison-001": {
+        "max_tokens": 8192,
+        "input_cost_per_token": 0.000004,
+        "output_cost_per_token": 0.000004,
+    },
+    "chat-bison-001": {
+        "max_tokens": 4096,
+        "input_cost_per_token": 0.000002,
+        "output_cost_per_token": 0.000002,
+    },
+    "command-nightly": {
+        "max_tokens": 4096,
+        "input_cost_per_token": 0.000015,
+        "output_cost_per_token": 0.000015,
+    },
+}
+
+
+class BerriSpendLogger:
+    # Class variables or attributes
+    def __init__(self):
+        # Instance variables
+        self.account_id = os.getenv("BERRISPEND_ACCOUNT_ID")
+
+    def price_calculator(self, model, response_obj, start_time, end_time):
+        # try and find if the model is in the model_cost map
+        # else default to the average of the costs
+        prompt_tokens_cost_usd_dollar = 0
+        completion_tokens_cost_usd_dollar = 0
+        if model in model_cost:
+            prompt_tokens_cost_usd_dollar = (
+                model_cost[model]["input_cost_per_token"]
+                * response_obj["usage"]["prompt_tokens"]
+            )
+            completion_tokens_cost_usd_dollar = (
+                model_cost[model]["output_cost_per_token"]
+                * response_obj["usage"]["completion_tokens"]
+            )
+        elif "replicate" in model:
+            # replicate models are charged based on time
+            # llama 2 runs on an nvidia a100 which costs $0.0032 per second - https://replicate.com/replicate/llama-2-70b-chat
+            model_run_time = end_time - start_time  # assuming time in seconds
+            cost_usd_dollar = model_run_time * 0.0032
+            prompt_tokens_cost_usd_dollar = cost_usd_dollar / 2
+            completion_tokens_cost_usd_dollar = cost_usd_dollar / 2
+        else:
+            # calculate average input cost
+            input_cost_sum = 0
+            output_cost_sum = 0
+            for model in model_cost:
+                input_cost_sum += model_cost[model]["input_cost_per_token"]
+                output_cost_sum += model_cost[model]["output_cost_per_token"]
+            avg_input_cost = input_cost_sum / len(model_cost.keys())
+            avg_output_cost = output_cost_sum / len(model_cost.keys())
+            prompt_tokens_cost_usd_dollar = (
+                model_cost[model]["input_cost_per_token"]
+                * response_obj["usage"]["prompt_tokens"]
+            )
+            completion_tokens_cost_usd_dollar = (
+                model_cost[model]["output_cost_per_token"]
+                * response_obj["usage"]["completion_tokens"]
+            )
+        return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
+
+    def log_event(
+        self, model, messages, response_obj, start_time, end_time, print_verbose
+    ):
+        # Method definition
+        try:
+            print_verbose(
+                f"BerriSpend Logging - Enters logging function for model {model}"
+            )
+
+            url = f"https://berrispend.berri.ai/spend"
+            headers = {"Content-Type": "application/json"}
+
+            (
+                prompt_tokens_cost_usd_dollar,
+                completion_tokens_cost_usd_dollar,
+            ) = self.price_calculator(model, response_obj, start_time, end_time)
+            total_cost = (
+                prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
+            )
+
+            response_time = (end_time - start_time).total_seconds()
+            if "response" in response_obj:
+                data = [
+                    {
+                        "response_time": response_time,
+                        "model_id": response_obj["model"],
+                        "total_cost": total_cost,
+                        "messages": messages,
+                        "response": response_obj["choices"][0]["message"]["content"],
+                        "account_id": self.account_id,
+                    }
+                ]
+            elif "error" in response_obj:
+                data = [
+                    {
+                        "response_time": response_time,
+                        "model_id": response_obj["model"],
+                        "total_cost": total_cost,
+                        "messages": messages,
+                        "error": response_obj["error"],
+                        "account_id": self.account_id,
+                    }
+                ]
+
+            print_verbose(f"BerriSpend Logging - final data object: {data}")
+            response = requests.post(url, headers=headers, json=data)
+        except:
+            # traceback.print_exc()
+            print_verbose(f"BerriSpend Logging Error - {traceback.format_exc()}")
+            pass

litellm/integrations/custom_logger.py

@@ -0,0 +1,130 @@
+#### What this does ####
+#    On success, logs events to Promptlayer
+import dotenv, os
+import requests
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.caching import DualCache
+from typing import Literal
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+
+
+class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
+    # Class variables or attributes
+    def __init__(self):
+        pass
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        pass
+
+    def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    def log_stream_event(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    #### ASYNC ####
+
+    async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    async def async_log_pre_api_call(self, model, messages, kwargs):
+        pass
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        pass
+
+    #### CALL HOOKS - proxy only ####
+    """
+    Control the modify incoming / outgoung data before calling the model
+    """
+
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        cache: DualCache,
+        data: dict,
+        call_type: Literal["completion", "embeddings"],
+    ):
+        pass
+
+    async def async_post_call_failure_hook(
+        self, original_exception: Exception, user_api_key_dict: UserAPIKeyAuth
+    ):
+        pass
+
+    #### SINGLE-USE #### - https://docs.litellm.ai/docs/observability/custom_callback#using-your-custom-callback-function
+
+    def log_input_event(self, model, messages, kwargs, print_verbose, callback_func):
+        try:
+            kwargs["model"] = model
+            kwargs["messages"] = messages
+            kwargs["log_event_type"] = "pre_api_call"
+            callback_func(
+                kwargs,
+            )
+            print_verbose(f"Custom Logger - model call details: {kwargs}")
+        except:
+            traceback.print_exc()
+            print_verbose(f"Custom Logger Error - {traceback.format_exc()}")
+
+    async def async_log_input_event(
+        self, model, messages, kwargs, print_verbose, callback_func
+    ):
+        try:
+            kwargs["model"] = model
+            kwargs["messages"] = messages
+            kwargs["log_event_type"] = "pre_api_call"
+            await callback_func(
+                kwargs,
+            )
+            print_verbose(f"Custom Logger - model call details: {kwargs}")
+        except:
+            traceback.print_exc()
+            print_verbose(f"Custom Logger Error - {traceback.format_exc()}")
+
+    def log_event(
+        self, kwargs, response_obj, start_time, end_time, print_verbose, callback_func
+    ):
+        # Method definition
+        try:
+            kwargs["log_event_type"] = "post_api_call"
+            callback_func(
+                kwargs,  # kwargs to func
+                response_obj,
+                start_time,
+                end_time,
+            )
+            print_verbose(f"Custom Logger - final response object: {response_obj}")
+        except:
+            # traceback.print_exc()
+            print_verbose(f"Custom Logger Error - {traceback.format_exc()}")
+            pass
+
+    async def async_log_event(
+        self, kwargs, response_obj, start_time, end_time, print_verbose, callback_func
+    ):
+        # Method definition
+        try:
+            kwargs["log_event_type"] = "post_api_call"
+            await callback_func(
+                kwargs,  # kwargs to func
+                response_obj,
+                start_time,
+                end_time,
+            )
+            print_verbose(f"Custom Logger - final response object: {response_obj}")
+        except:
+            # traceback.print_exc()
+            print_verbose(f"Custom Logger Error - {traceback.format_exc()}")
+            pass

litellm/integrations/dynamodb.py

@@ -0,0 +1,92 @@
+#### What this does ####
+#    On success + failure, log events to Supabase
+
+import dotenv, os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+import datetime, subprocess, sys
+import litellm, uuid
+from litellm._logging import print_verbose
+
+
+class DyanmoDBLogger:
+    # Class variables or attributes
+
+    def __init__(self):
+        # Instance variables
+        import boto3
+
+        self.dynamodb = boto3.resource(
+            "dynamodb", region_name=os.environ["AWS_REGION_NAME"]
+        )
+        if litellm.dynamodb_table_name is None:
+            raise ValueError(
+                "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
+            )
+        self.table_name = litellm.dynamodb_table_name
+
+    async def _async_log_event(
+        self, kwargs, response_obj, start_time, end_time, print_verbose
+    ):
+        self.log_event(kwargs, response_obj, start_time, end_time, print_verbose)
+
+    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+        try:
+            print_verbose(
+                f"DynamoDB Logging - Enters logging function for model {kwargs}"
+            )
+
+            # construct payload to send to DynamoDB
+            # follows the same params as langfuse.py
+            litellm_params = kwargs.get("litellm_params", {})
+            metadata = (
+                litellm_params.get("metadata", {}) or {}
+            )  # if litellm_params['metadata'] == None
+            messages = kwargs.get("messages")
+            optional_params = kwargs.get("optional_params", {})
+            call_type = kwargs.get("call_type", "litellm.completion")
+            usage = response_obj["usage"]
+            id = response_obj.get("id", str(uuid.uuid4()))
+
+            # Build the initial payload
+            payload = {
+                "id": id,
+                "call_type": call_type,
+                "startTime": start_time,
+                "endTime": end_time,
+                "model": kwargs.get("model", ""),
+                "user": kwargs.get("user", ""),
+                "modelParameters": optional_params,
+                "messages": messages,
+                "response": response_obj,
+                "usage": usage,
+                "metadata": metadata,
+            }
+
+            # Ensure everything in the payload is converted to str
+            for key, value in payload.items():
+                try:
+                    payload[key] = str(value)
+                except:
+                    # non blocking if it can't cast to a str
+                    pass
+
+            print_verbose(f"\nDynamoDB Logger - Logging payload = {payload}")
+
+            # put data in dyanmo DB
+            table = self.dynamodb.Table(self.table_name)
+            # Assuming log_data is a dictionary with log information
+            response = table.put_item(Item=payload)
+
+            print_verbose(f"Response from DynamoDB:{str(response)}")
+
+            print_verbose(
+                f"DynamoDB Layer Logging - final response object: {response_obj}"
+            )
+            return response
+        except:
+            traceback.print_exc()
+            print_verbose(f"DynamoDB Layer Error - {traceback.format_exc()}")
+            pass

litellm/integrations/helicone.py

@@ -0,0 +1,114 @@
+#### What this does ####
+#    On success, logs events to Helicone
+import dotenv, os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+
+
+class HeliconeLogger:
+    # Class variables or attributes
+    helicone_model_list = ["gpt", "claude"]
+
+    def __init__(self):
+        # Instance variables
+        self.provider_url = "https://api.openai.com/v1"
+        self.key = os.getenv("HELICONE_API_KEY")
+
+    def claude_mapping(self, model, messages, response_obj):
+        from anthropic import HUMAN_PROMPT, AI_PROMPT
+
+        prompt = f"{HUMAN_PROMPT}"
+        for message in messages:
+            if "role" in message:
+                if message["role"] == "user":
+                    prompt += f"{HUMAN_PROMPT}{message['content']}"
+                else:
+                    prompt += f"{AI_PROMPT}{message['content']}"
+            else:
+                prompt += f"{HUMAN_PROMPT}{message['content']}"
+        prompt += f"{AI_PROMPT}"
+        claude_provider_request = {"model": model, "prompt": prompt}
+
+        claude_response_obj = {
+            "completion": response_obj["choices"][0]["message"]["content"],
+            "model": model,
+            "stop_reason": "stop_sequence",
+        }
+
+        return claude_provider_request, claude_response_obj
+
+    def log_success(
+        self, model, messages, response_obj, start_time, end_time, print_verbose
+    ):
+        # Method definition
+        try:
+            print_verbose(
+                f"Helicone Logging - Enters logging function for model {model}"
+            )
+            model = (
+                model
+                if any(
+                    accepted_model in model
+                    for accepted_model in self.helicone_model_list
+                )
+                else "gpt-3.5-turbo"
+            )
+            provider_request = {"model": model, "messages": messages}
+
+            if "claude" in model:
+                provider_request, response_obj = self.claude_mapping(
+                    model=model, messages=messages, response_obj=response_obj
+                )
+
+            providerResponse = {
+                "json": response_obj,
+                "headers": {"openai-version": "2020-10-01"},
+                "status": 200,
+            }
+
+            # Code to be executed
+            url = "https://api.hconeai.com/oai/v1/log"
+            headers = {
+                "Authorization": f"Bearer {self.key}",
+                "Content-Type": "application/json",
+            }
+            start_time_seconds = int(start_time.timestamp())
+            start_time_milliseconds = int(
+                (start_time.timestamp() - start_time_seconds) * 1000
+            )
+            end_time_seconds = int(end_time.timestamp())
+            end_time_milliseconds = int(
+                (end_time.timestamp() - end_time_seconds) * 1000
+            )
+            data = {
+                "providerRequest": {
+                    "url": self.provider_url,
+                    "json": provider_request,
+                    "meta": {"Helicone-Auth": f"Bearer {self.key}"},
+                },
+                "providerResponse": providerResponse,
+                "timing": {
+                    "startTime": {
+                        "seconds": start_time_seconds,
+                        "milliseconds": start_time_milliseconds,
+                    },
+                    "endTime": {
+                        "seconds": end_time_seconds,
+                        "milliseconds": end_time_milliseconds,
+                    },
+                },  # {"seconds": .., "milliseconds": ..}
+            }
+            response = requests.post(url, headers=headers, json=data)
+            if response.status_code == 200:
+                print_verbose("Helicone Logging - Success!")
+            else:
+                print_verbose(
+                    f"Helicone Logging - Error Request was not successful. Status Code: {response.status_code}"
+                )
+                print_verbose(f"Helicone Logging - Error {response.text}")
+        except:
+            # traceback.print_exc()
+            print_verbose(f"Helicone Logging Error - {traceback.format_exc()}")
+            pass

litellm/integrations/langfuse.py

@@ -0,0 +1,191 @@
+#### What this does ####
+#    On success, logs events to Langfuse
+import dotenv, os
+import requests
+import requests
+from datetime import datetime
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+from packaging.version import Version
+
+
+class LangFuseLogger:
+    # Class variables or attributes
+    def __init__(self):
+        try:
+            from langfuse import Langfuse
+        except Exception as e:
+            raise Exception(
+                f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\033[0m"
+            )
+        # Instance variables
+        self.secret_key = os.getenv("LANGFUSE_SECRET_KEY")
+        self.public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
+        self.langfuse_host = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
+        self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
+        self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
+        self.Langfuse = Langfuse(
+            public_key=self.public_key,
+            secret_key=self.secret_key,
+            host=self.langfuse_host,
+            release=self.langfuse_release,
+            debug=self.langfuse_debug,
+        )
+
+    def log_event(
+        self, kwargs, response_obj, start_time, end_time, user_id, print_verbose
+    ):
+        # Method definition
+
+        try:
+            print_verbose(
+                f"Langfuse Logging - Enters logging function for model {kwargs}"
+            )
+            litellm_params = kwargs.get("litellm_params", {})
+            metadata = (
+                litellm_params.get("metadata", {}) or {}
+            )  # if litellm_params['metadata'] == None
+            prompt = [kwargs.get("messages")]
+            optional_params = kwargs.get("optional_params", {})
+
+            optional_params.pop("functions", None)
+            optional_params.pop("tools", None)
+
+            # langfuse only accepts str, int, bool, float for logging
+            for param, value in optional_params.items():
+                if not isinstance(value, (str, int, bool, float)):
+                    try:
+                        optional_params[param] = str(value)
+                    except:
+                        # if casting value to str fails don't block logging
+                        pass
+
+            # end of processing langfuse ########################
+            input = prompt
+            output = response_obj["choices"][0]["message"].json()
+            print_verbose(
+                f"OUTPUT IN LANGFUSE: {output}; original: {response_obj['choices'][0]['message']}"
+            )
+            self._log_langfuse_v2(
+                user_id,
+                metadata,
+                output,
+                start_time,
+                end_time,
+                kwargs,
+                optional_params,
+                input,
+                response_obj,
+            ) if self._is_langfuse_v2() else self._log_langfuse_v1(
+                user_id,
+                metadata,
+                output,
+                start_time,
+                end_time,
+                kwargs,
+                optional_params,
+                input,
+                response_obj,
+            )
+
+            self.Langfuse.flush()
+            print_verbose(
+                f"Langfuse Layer Logging - final response object: {response_obj}"
+            )
+        except:
+            traceback.print_exc()
+            print_verbose(f"Langfuse Layer Error - {traceback.format_exc()}")
+            pass
+
+    async def _async_log_event(
+        self, kwargs, response_obj, start_time, end_time, user_id, print_verbose
+    ):
+        self.log_event(
+            kwargs, response_obj, start_time, end_time, user_id, print_verbose
+        )
+
+    def _is_langfuse_v2(self):
+        import langfuse
+
+        return Version(langfuse.version.__version__) >= Version("2.0.0")
+
+    def _log_langfuse_v1(
+        self,
+        user_id,
+        metadata,
+        output,
+        start_time,
+        end_time,
+        kwargs,
+        optional_params,
+        input,
+        response_obj,
+    ):
+        from langfuse.model import CreateTrace, CreateGeneration
+
+        print(
+            "Please upgrade langfuse to v2.0.0 or higher: https://github.com/langfuse/langfuse-python/releases/tag/v2.0.1"
+        )
+
+        trace = self.Langfuse.trace(
+            CreateTrace(
+                name=metadata.get("generation_name", "litellm-completion"),
+                input=input,
+                output=output,
+                userId=user_id,
+            )
+        )
+
+        trace.generation(
+            CreateGeneration(
+                name=metadata.get("generation_name", "litellm-completion"),
+                startTime=start_time,
+                endTime=end_time,
+                model=kwargs["model"],
+                modelParameters=optional_params,
+                input=input,
+                output=output,
+                usage={
+                    "prompt_tokens": response_obj["usage"]["prompt_tokens"],
+                    "completion_tokens": response_obj["usage"]["completion_tokens"],
+                },
+                metadata=metadata,
+            )
+        )
+
+    def _log_langfuse_v2(
+        self,
+        user_id,
+        metadata,
+        output,
+        start_time,
+        end_time,
+        kwargs,
+        optional_params,
+        input,
+        response_obj,
+    ):
+        trace = self.Langfuse.trace(
+            name=metadata.get("generation_name", "litellm-completion"),
+            input=input,
+            output=output,
+            user_id=metadata.get("trace_user_id", user_id),
+            id=metadata.get("trace_id", None),
+        )
+
+        trace.generation(
+            name=metadata.get("generation_name", "litellm-completion"),
+            id=metadata.get("generation_id", None),
+            startTime=start_time,
+            endTime=end_time,
+            model=kwargs["model"],
+            modelParameters=optional_params,
+            input=input,
+            output=output,
+            usage={
+                "prompt_tokens": response_obj["usage"]["prompt_tokens"],
+                "completion_tokens": response_obj["usage"]["completion_tokens"],
+            },
+            metadata=metadata,
+        )

litellm/integrations/langsmith.py

@@ -0,0 +1,75 @@
+#### What this does ####
+#    On success, logs events to Langsmith
+import dotenv, os
+import requests
+import requests
+from datetime import datetime
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+
+
+class LangsmithLogger:
+    # Class variables or attributes
+    def __init__(self):
+        self.langsmith_api_key = os.getenv("LANGSMITH_API_KEY")
+
+    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+        # Method definition
+        # inspired by Langsmith http api here: https://github.com/langchain-ai/langsmith-cookbook/blob/main/tracing-examples/rest/rest.ipynb
+        metadata = {}
+        if "litellm_params" in kwargs:
+            metadata = kwargs["litellm_params"].get("metadata", {})
+        # set project name and run_name for langsmith logging
+        # users can pass project_name and run name to litellm.completion()
+        # Example: litellm.completion(model, messages, metadata={"project_name": "my-litellm-project", "run_name": "my-langsmith-run"})
+        # if not set litellm will use default project_name = litellm-completion, run_name = LLMRun
+        project_name = metadata.get("project_name", "litellm-completion")
+        run_name = metadata.get("run_name", "LLMRun")
+        print_verbose(
+            f"Langsmith Logging - project_name: {project_name}, run_name {run_name}"
+        )
+        try:
+            print_verbose(
+                f"Langsmith Logging - Enters logging function for model {kwargs}"
+            )
+            import requests
+            import datetime
+            from datetime import timezone
+
+            try:
+                start_time = kwargs["start_time"].astimezone(timezone.utc).isoformat()
+                end_time = kwargs["end_time"].astimezone(timezone.utc).isoformat()
+            except:
+                start_time = datetime.datetime.utcnow().isoformat()
+                end_time = datetime.datetime.utcnow().isoformat()
+
+            # filter out kwargs to not include any dicts, langsmith throws an erros when trying to log kwargs
+            new_kwargs = {}
+            for key in kwargs:
+                value = kwargs[key]
+                if key == "start_time" or key == "end_time":
+                    pass
+                elif type(value) != dict:
+                    new_kwargs[key] = value
+
+            requests.post(
+                "https://api.smith.langchain.com/runs",
+                json={
+                    "name": run_name,
+                    "run_type": "llm",  # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain"
+                    "inputs": {**new_kwargs},
+                    "outputs": response_obj.json(),
+                    "session_name": project_name,
+                    "start_time": start_time,
+                    "end_time": end_time,
+                },
+                headers={"x-api-key": self.langsmith_api_key},
+            )
+            print_verbose(
+                f"Langsmith Layer Logging - final response object: {response_obj}"
+            )
+        except:
+            # traceback.print_exc()
+            print_verbose(f"Langsmith Layer Error - {traceback.format_exc()}")
+            pass

litellm/integrations/litedebugger.py

@@ -0,0 +1,262 @@
+import requests, traceback, json, os
+import types
+
+
+class LiteDebugger:
+    user_email = None
+    dashboard_url = None
+
+    def __init__(self, email=None):
+        self.api_url = "https://api.litellm.ai/debugger"
+        self.validate_environment(email)
+        pass
+
+    def validate_environment(self, email):
+        try:
+            self.user_email = (
+                email or os.getenv("LITELLM_TOKEN") or os.getenv("LITELLM_EMAIL")
+            )
+            if (
+                self.user_email == None
+            ):  # if users are trying to use_client=True but token not set
+                raise ValueError(
+                    "litellm.use_client = True but no token or email passed. Please set it in litellm.token"
+                )
+            self.dashboard_url = "https://admin.litellm.ai/" + self.user_email
+            try:
+                print(
+                    f"\033[92mHere's your LiteLLM Dashboard 👉 \033[94m\033[4m{self.dashboard_url}\033[0m"
+                )
+            except:
+                print(f"Here's your LiteLLM Dashboard 👉 {self.dashboard_url}")
+            if self.user_email == None:
+                raise ValueError(
+                    "[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_TOKEN. Set it in your environment. Eg.: os.environ['LITELLM_TOKEN']= <your_email>"
+                )
+        except Exception as e:
+            raise ValueError(
+                "[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_TOKEN. Set it in your environment. Eg.: os.environ['LITELLM_TOKEN']= <your_email>"
+            )
+
+    def input_log_event(
+        self,
+        model,
+        messages,
+        end_user,
+        litellm_call_id,
+        call_type,
+        print_verbose,
+        litellm_params,
+        optional_params,
+    ):
+        print_verbose(
+            f"LiteDebugger: Pre-API Call Logging for call id {litellm_call_id}"
+        )
+        try:
+            print_verbose(
+                f"LiteLLMDebugger: Logging - Enters input logging function for model {model}"
+            )
+
+            def remove_key_value(dictionary, key):
+                new_dict = dictionary.copy()  # Create a copy of the original dictionary
+                new_dict.pop(key)  # Remove the specified key-value pair from the copy
+                return new_dict
+
+            updated_litellm_params = remove_key_value(litellm_params, "logger_fn")
+
+            if call_type == "embedding":
+                for (
+                    message
+                ) in (
+                    messages
+                ):  # assuming the input is a list as required by the embedding function
+                    litellm_data_obj = {
+                        "model": model,
+                        "messages": [{"role": "user", "content": message}],
+                        "end_user": end_user,
+                        "status": "initiated",
+                        "litellm_call_id": litellm_call_id,
+                        "user_email": self.user_email,
+                        "litellm_params": updated_litellm_params,
+                        "optional_params": optional_params,
+                    }
+                    print_verbose(
+                        f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}"
+                    )
+                    response = requests.post(
+                        url=self.api_url,
+                        headers={"content-type": "application/json"},
+                        data=json.dumps(litellm_data_obj),
+                    )
+                print_verbose(f"LiteDebugger: embedding api response - {response.text}")
+            elif call_type == "completion":
+                litellm_data_obj = {
+                    "model": model,
+                    "messages": messages
+                    if isinstance(messages, list)
+                    else [{"role": "user", "content": messages}],
+                    "end_user": end_user,
+                    "status": "initiated",
+                    "litellm_call_id": litellm_call_id,
+                    "user_email": self.user_email,
+                    "litellm_params": updated_litellm_params,
+                    "optional_params": optional_params,
+                }
+                print_verbose(
+                    f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}"
+                )
+                response = requests.post(
+                    url=self.api_url,
+                    headers={"content-type": "application/json"},
+                    data=json.dumps(litellm_data_obj),
+                )
+                print_verbose(
+                    f"LiteDebugger: completion api response - {response.text}"
+                )
+        except:
+            print_verbose(
+                f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}"
+            )
+            pass
+
+    def post_call_log_event(
+        self, original_response, litellm_call_id, print_verbose, call_type, stream
+    ):
+        print_verbose(
+            f"LiteDebugger: Post-API Call Logging for call id {litellm_call_id}"
+        )
+        try:
+            if call_type == "embedding":
+                litellm_data_obj = {
+                    "status": "received",
+                    "additional_details": {
+                        "original_response": str(
+                            original_response["data"][0]["embedding"][:5]
+                        )
+                    },  # don't store the entire vector
+                    "litellm_call_id": litellm_call_id,
+                    "user_email": self.user_email,
+                }
+            elif call_type == "completion" and not stream:
+                litellm_data_obj = {
+                    "status": "received",
+                    "additional_details": {"original_response": original_response},
+                    "litellm_call_id": litellm_call_id,
+                    "user_email": self.user_email,
+                }
+            elif call_type == "completion" and stream:
+                litellm_data_obj = {
+                    "status": "received",
+                    "additional_details": {
+                        "original_response": "Streamed response"
+                        if isinstance(original_response, types.GeneratorType)
+                        else original_response
+                    },
+                    "litellm_call_id": litellm_call_id,
+                    "user_email": self.user_email,
+                }
+            print_verbose(f"litedebugger post-call data object - {litellm_data_obj}")
+            response = requests.post(
+                url=self.api_url,
+                headers={"content-type": "application/json"},
+                data=json.dumps(litellm_data_obj),
+            )
+            print_verbose(f"LiteDebugger: api response - {response.text}")
+        except:
+            print_verbose(
+                f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}"
+            )
+
+    def log_event(
+        self,
+        end_user,
+        response_obj,
+        start_time,
+        end_time,
+        litellm_call_id,
+        print_verbose,
+        call_type,
+        stream=False,
+    ):
+        print_verbose(
+            f"LiteDebugger: Success/Failure Call Logging for call id {litellm_call_id}"
+        )
+        try:
+            print_verbose(
+                f"LiteLLMDebugger: Success/Failure Logging - Enters handler logging function for function {call_type} and stream set to {stream} with response object {response_obj}"
+            )
+            total_cost = 0  # [TODO] implement cost tracking
+            response_time = (end_time - start_time).total_seconds()
+            if call_type == "completion" and stream == False:
+                litellm_data_obj = {
+                    "response_time": response_time,
+                    "total_cost": total_cost,
+                    "response": response_obj["choices"][0]["message"]["content"],
+                    "litellm_call_id": litellm_call_id,
+                    "status": "success",
+                }
+                print_verbose(
+                    f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
+                )
+                response = requests.post(
+                    url=self.api_url,
+                    headers={"content-type": "application/json"},
+                    data=json.dumps(litellm_data_obj),
+                )
+            elif call_type == "embedding":
+                litellm_data_obj = {
+                    "response_time": response_time,
+                    "total_cost": total_cost,
+                    "response": str(response_obj["data"][0]["embedding"][:5]),
+                    "litellm_call_id": litellm_call_id,
+                    "status": "success",
+                }
+                response = requests.post(
+                    url=self.api_url,
+                    headers={"content-type": "application/json"},
+                    data=json.dumps(litellm_data_obj),
+                )
+            elif call_type == "completion" and stream == True:
+                if len(response_obj["content"]) > 0:  # don't log the empty strings
+                    litellm_data_obj = {
+                        "response_time": response_time,
+                        "total_cost": total_cost,
+                        "response": response_obj["content"],
+                        "litellm_call_id": litellm_call_id,
+                        "status": "success",
+                    }
+                    print_verbose(
+                        f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
+                    )
+                    response = requests.post(
+                        url=self.api_url,
+                        headers={"content-type": "application/json"},
+                        data=json.dumps(litellm_data_obj),
+                    )
+            elif "error" in response_obj:
+                if "Unable to map your input to a model." in response_obj["error"]:
+                    total_cost = 0
+                litellm_data_obj = {
+                    "response_time": response_time,
+                    "model": response_obj["model"],
+                    "total_cost": total_cost,
+                    "error": response_obj["error"],
+                    "end_user": end_user,
+                    "litellm_call_id": litellm_call_id,
+                    "status": "failure",
+                    "user_email": self.user_email,
+                }
+                print_verbose(
+                    f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
+                )
+                response = requests.post(
+                    url=self.api_url,
+                    headers={"content-type": "application/json"},
+                    data=json.dumps(litellm_data_obj),
+                )
+                print_verbose(f"LiteDebugger: api response - {response.text}")
+        except:
+            print_verbose(
+                f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}"
+            )
+            pass

litellm/integrations/llmonitor.py

@@ -0,0 +1,127 @@
+#### What this does ####
+#    On success + failure, log events to aispend.io
+import datetime
+import traceback
+import dotenv
+import os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+
+
+# convert to {completion: xx, tokens: xx}
+def parse_usage(usage):
+    return {
+        "completion": usage["completion_tokens"] if "completion_tokens" in usage else 0,
+        "prompt": usage["prompt_tokens"] if "prompt_tokens" in usage else 0,
+    }
+
+
+def parse_messages(input):
+    if input is None:
+        return None
+
+    def clean_message(message):
+        # if is strin, return as is
+        if isinstance(message, str):
+            return message
+
+        if "message" in message:
+            return clean_message(message["message"])
+        text = message["content"]
+        if text == None:
+            text = message.get("function_call", None)
+
+        return {
+            "role": message["role"],
+            "text": text,
+        }
+
+    if isinstance(input, list):
+        if len(input) == 1:
+            return clean_message(input[0])
+        else:
+            return [clean_message(msg) for msg in input]
+    else:
+        return clean_message(input)
+
+
+class LLMonitorLogger:
+    # Class variables or attributes
+    def __init__(self):
+        # Instance variables
+        self.api_url = os.getenv("LLMONITOR_API_URL") or "https://app.llmonitor.com"
+        self.app_id = os.getenv("LLMONITOR_APP_ID")
+
+    def log_event(
+        self,
+        type,
+        event,
+        run_id,
+        model,
+        print_verbose,
+        input=None,
+        user_id=None,
+        response_obj=None,
+        start_time=datetime.datetime.now(),
+        end_time=datetime.datetime.now(),
+        error=None,
+    ):
+        # Method definition
+        try:
+            print_verbose(f"LLMonitor Logging - Logging request for model {model}")
+
+            if response_obj:
+                usage = (
+                    parse_usage(response_obj["usage"])
+                    if "usage" in response_obj
+                    else None
+                )
+                output = response_obj["choices"] if "choices" in response_obj else None
+            else:
+                usage = None
+                output = None
+
+            if error:
+                error_obj = {"stack": error}
+
+            else:
+                error_obj = None
+
+            data = [
+                {
+                    "type": type,
+                    "name": model,
+                    "runId": run_id,
+                    "app": self.app_id,
+                    "event": "start",
+                    "timestamp": start_time.isoformat(),
+                    "userId": user_id,
+                    "input": parse_messages(input),
+                },
+                {
+                    "type": type,
+                    "runId": run_id,
+                    "app": self.app_id,
+                    "event": event,
+                    "error": error_obj,
+                    "timestamp": end_time.isoformat(),
+                    "userId": user_id,
+                    "output": parse_messages(output),
+                    "tokensUsage": usage,
+                },
+            ]
+
+            print_verbose(f"LLMonitor Logging - final data object: {data}")
+
+            response = requests.post(
+                self.api_url + "/api/report",
+                headers={"Content-Type": "application/json"},
+                json={"events": data},
+            )
+
+            print_verbose(f"LLMonitor Logging - response: {response}")
+        except:
+            # traceback.print_exc()
+            print_verbose(f"LLMonitor Logging Error - {traceback.format_exc()}")
+            pass

litellm/integrations/prompt_layer.py

@@ -0,0 +1,72 @@
+#### What this does ####
+#    On success, logs events to Promptlayer
+import dotenv, os
+import requests
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+
+
+class PromptLayerLogger:
+    # Class variables or attributes
+    def __init__(self):
+        # Instance variables
+        self.key = os.getenv("PROMPTLAYER_API_KEY")
+
+    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+        # Method definition
+        try:
+            new_kwargs = {}
+            new_kwargs["model"] = kwargs["model"]
+            new_kwargs["messages"] = kwargs["messages"]
+
+            # add kwargs["optional_params"] to new_kwargs
+            for optional_param in kwargs["optional_params"]:
+                new_kwargs[optional_param] = kwargs["optional_params"][optional_param]
+
+            print_verbose(
+                f"Prompt Layer Logging - Enters logging function for model kwargs: {new_kwargs}\n, response: {response_obj}"
+            )
+
+            request_response = requests.post(
+                "https://api.promptlayer.com/rest/track-request",
+                json={
+                    "function_name": "openai.ChatCompletion.create",
+                    "kwargs": new_kwargs,
+                    "tags": ["hello", "world"],
+                    "request_response": dict(response_obj),
+                    "request_start_time": int(start_time.timestamp()),
+                    "request_end_time": int(end_time.timestamp()),
+                    "api_key": self.key,
+                    # Optional params for PromptLayer
+                    # "prompt_id": "<PROMPT ID>",
+                    # "prompt_input_variables": "<Dictionary of variables for prompt>",
+                    # "prompt_version":1,
+                },
+            )
+            print_verbose(
+                f"Prompt Layer Logging: success - final response object: {request_response.text}"
+            )
+            response_json = request_response.json()
+            if "success" not in request_response.json():
+                raise Exception("Promptlayer did not successfully log the response!")
+
+            if "request_id" in response_json:
+                print(kwargs["litellm_params"]["metadata"])
+                if kwargs["litellm_params"]["metadata"] is not None:
+                    response = requests.post(
+                        "https://api.promptlayer.com/rest/track-metadata",
+                        json={
+                            "request_id": response_json["request_id"],
+                            "api_key": self.key,
+                            "metadata": kwargs["litellm_params"]["metadata"],
+                        },
+                    )
+                    print_verbose(
+                        f"Prompt Layer Logging: success - metadata post response object: {response.text}"
+                    )
+
+        except:
+            print_verbose(f"error: Prompt Layer Error - {traceback.format_exc()}")
+            pass

litellm/integrations/s3.py

@@ -0,0 +1,150 @@
+#### What this does ####
+#    On success + failure, log events to Supabase
+
+import dotenv, os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+import datetime, subprocess, sys
+import litellm, uuid
+from litellm._logging import print_verbose
+
+
+class S3Logger:
+    # Class variables or attributes
+    def __init__(
+        self,
+        s3_bucket_name=None,
+        s3_region_name=None,
+        s3_api_version=None,
+        s3_use_ssl=True,
+        s3_verify=None,
+        s3_endpoint_url=None,
+        s3_aws_access_key_id=None,
+        s3_aws_secret_access_key=None,
+        s3_aws_session_token=None,
+        s3_config=None,
+        **kwargs,
+    ):
+        import boto3
+
+        try:
+            print_verbose("in init s3 logger")
+
+            if litellm.s3_callback_params is not None:
+                # read in .env variables - example os.environ/AWS_BUCKET_NAME
+                for key, value in litellm.s3_callback_params.items():
+                    if type(value) is str and value.startswith("os.environ/"):
+                        litellm.s3_callback_params[key] = litellm.get_secret(value)
+                # now set s3 params from litellm.s3_logger_params
+                s3_bucket_name = litellm.s3_callback_params.get("s3_bucket_name")
+                s3_region_name = litellm.s3_callback_params.get("s3_region_name")
+                s3_api_version = litellm.s3_callback_params.get("s3_api_version")
+                s3_use_ssl = litellm.s3_callback_params.get("s3_use_ssl")
+                s3_verify = litellm.s3_callback_params.get("s3_verify")
+                s3_endpoint_url = litellm.s3_callback_params.get("s3_endpoint_url")
+                s3_aws_access_key_id = litellm.s3_callback_params.get(
+                    "s3_aws_access_key_id"
+                )
+                s3_aws_secret_access_key = litellm.s3_callback_params.get(
+                    "s3_aws_secret_access_key"
+                )
+                s3_aws_session_token = litellm.s3_callback_params.get(
+                    "s3_aws_session_token"
+                )
+                s3_config = litellm.s3_callback_params.get("s3_config")
+                # done reading litellm.s3_callback_params
+
+            self.bucket_name = s3_bucket_name
+            # Create an S3 client with custom endpoint URL
+            self.s3_client = boto3.client(
+                "s3",
+                region_name=s3_region_name,
+                endpoint_url=s3_endpoint_url,
+                api_version=s3_api_version,
+                use_ssl=s3_use_ssl,
+                verify=s3_verify,
+                aws_access_key_id=s3_aws_access_key_id,
+                aws_secret_access_key=s3_aws_secret_access_key,
+                aws_session_token=s3_aws_session_token,
+                config=s3_config,
+                **kwargs,
+            )
+        except Exception as e:
+            print_verbose(f"Got exception on init s3 client {str(e)}")
+            raise e
+
+    async def _async_log_event(
+        self, kwargs, response_obj, start_time, end_time, print_verbose
+    ):
+        self.log_event(kwargs, response_obj, start_time, end_time, print_verbose)
+
+    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+        try:
+            print_verbose(f"s3 Logging - Enters logging function for model {kwargs}")
+
+            # construct payload to send to s3
+            # follows the same params as langfuse.py
+            litellm_params = kwargs.get("litellm_params", {})
+            metadata = (
+                litellm_params.get("metadata", {}) or {}
+            )  # if litellm_params['metadata'] == None
+            messages = kwargs.get("messages")
+            optional_params = kwargs.get("optional_params", {})
+            call_type = kwargs.get("call_type", "litellm.completion")
+            cache_hit = kwargs.get("cache_hit", False)
+            usage = response_obj["usage"]
+            id = response_obj.get("id", str(uuid.uuid4()))
+
+            # Build the initial payload
+            payload = {
+                "id": id,
+                "call_type": call_type,
+                "cache_hit": cache_hit,
+                "startTime": start_time,
+                "endTime": end_time,
+                "model": kwargs.get("model", ""),
+                "user": kwargs.get("user", ""),
+                "modelParameters": optional_params,
+                "messages": messages,
+                "response": response_obj,
+                "usage": usage,
+                "metadata": metadata,
+            }
+
+            # Ensure everything in the payload is converted to str
+            for key, value in payload.items():
+                try:
+                    payload[key] = str(value)
+                except:
+                    # non blocking if it can't cast to a str
+                    pass
+
+            s3_object_key = (
+                payload["id"] + "-time=" + str(start_time)
+            )  # we need the s3 key to include the time, so we log cache hits too
+
+            import json
+
+            payload = json.dumps(payload)
+
+            print_verbose(f"\ns3 Logger - Logging payload = {payload}")
+
+            response = self.s3_client.put_object(
+                Bucket=self.bucket_name,
+                Key=s3_object_key,
+                Body=payload,
+                ContentType="application/json",
+                ContentLanguage="en",
+                ContentDisposition=f'inline; filename="{key}.json"',
+            )
+
+            print_verbose(f"Response from s3:{str(response)}")
+
+            print_verbose(f"s3 Layer Logging - final response object: {response_obj}")
+            return response
+        except Exception as e:
+            traceback.print_exc()
+            print_verbose(f"s3 Layer Error - {str(e)}\n{traceback.format_exc()}")
+            pass

litellm/integrations/supabase.py

@@ -0,0 +1,117 @@
+#### What this does ####
+#    On success + failure, log events to Supabase
+
+import dotenv, os
+import requests
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+import datetime, subprocess, sys
+import litellm
+
+
+class Supabase:
+    # Class variables or attributes
+    supabase_table_name = "request_logs"
+
+    def __init__(self):
+        # Instance variables
+        self.supabase_url = os.getenv("SUPABASE_URL")
+        self.supabase_key = os.getenv("SUPABASE_KEY")
+        try:
+            import supabase
+        except ImportError:
+            subprocess.check_call([sys.executable, "-m", "pip", "install", "supabase"])
+            import supabase
+        self.supabase_client = supabase.create_client(
+            self.supabase_url, self.supabase_key
+        )
+
+    def input_log_event(
+        self, model, messages, end_user, litellm_call_id, print_verbose
+    ):
+        try:
+            print_verbose(
+                f"Supabase Logging - Enters input logging function for model {model}"
+            )
+            supabase_data_obj = {
+                "model": model,
+                "messages": messages,
+                "end_user": end_user,
+                "status": "initiated",
+                "litellm_call_id": litellm_call_id,
+            }
+            data, count = (
+                self.supabase_client.table(self.supabase_table_name)
+                .insert(supabase_data_obj)
+                .execute()
+            )
+            print_verbose(f"data: {data}")
+        except:
+            print_verbose(f"Supabase Logging Error - {traceback.format_exc()}")
+            pass
+
+    def log_event(
+        self,
+        model,
+        messages,
+        end_user,
+        response_obj,
+        start_time,
+        end_time,
+        litellm_call_id,
+        print_verbose,
+    ):
+        try:
+            print_verbose(
+                f"Supabase Logging - Enters logging function for model {model}, response_obj: {response_obj}"
+            )
+
+            total_cost = litellm.completion_cost(completion_response=response_obj)
+
+            response_time = (end_time - start_time).total_seconds()
+            if "choices" in response_obj:
+                supabase_data_obj = {
+                    "response_time": response_time,
+                    "model": response_obj["model"],
+                    "total_cost": total_cost,
+                    "messages": messages,
+                    "response": response_obj["choices"][0]["message"]["content"],
+                    "end_user": end_user,
+                    "litellm_call_id": litellm_call_id,
+                    "status": "success",
+                }
+                print_verbose(
+                    f"Supabase Logging - final data object: {supabase_data_obj}"
+                )
+                data, count = (
+                    self.supabase_client.table(self.supabase_table_name)
+                    .upsert(supabase_data_obj, on_conflict="litellm_call_id")
+                    .execute()
+                )
+            elif "error" in response_obj:
+                if "Unable to map your input to a model." in response_obj["error"]:
+                    total_cost = 0
+                supabase_data_obj = {
+                    "response_time": response_time,
+                    "model": response_obj["model"],
+                    "total_cost": total_cost,
+                    "messages": messages,
+                    "error": response_obj["error"],
+                    "end_user": end_user,
+                    "litellm_call_id": litellm_call_id,
+                    "status": "failure",
+                }
+                print_verbose(
+                    f"Supabase Logging - final data object: {supabase_data_obj}"
+                )
+                data, count = (
+                    self.supabase_client.table(self.supabase_table_name)
+                    .upsert(supabase_data_obj, on_conflict="litellm_call_id")
+                    .execute()
+                )
+
+        except:
+            # traceback.print_exc()
+            print_verbose(f"Supabase Logging Error - {traceback.format_exc()}")
+            pass

litellm/integrations/traceloop.py

@@ -0,0 +1,114 @@
+class TraceloopLogger:
+    def __init__(self):
+        from traceloop.sdk.tracing.tracing import TracerWrapper
+        from traceloop.sdk import Traceloop
+
+        Traceloop.init(app_name="Litellm-Server", disable_batch=True)
+        self.tracer_wrapper = TracerWrapper()
+
+    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+        from opentelemetry.trace import SpanKind
+        from opentelemetry.semconv.ai import SpanAttributes
+
+        try:
+            tracer = self.tracer_wrapper.get_tracer()
+
+            model = kwargs.get("model")
+
+            # LiteLLM uses the standard OpenAI library, so it's already handled by Traceloop SDK
+            if kwargs.get("litellm_params").get("custom_llm_provider") == "openai":
+                return
+
+            optional_params = kwargs.get("optional_params", {})
+            with tracer.start_as_current_span(
+                "litellm.completion",
+                kind=SpanKind.CLIENT,
+            ) as span:
+                if span.is_recording():
+                    span.set_attribute(
+                        SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")
+                    )
+                    if "stop" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_CHAT_STOP_SEQUENCES,
+                            optional_params.get("stop"),
+                        )
+                    if "frequency_penalty" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_FREQUENCY_PENALTY,
+                            optional_params.get("frequency_penalty"),
+                        )
+                    if "presence_penalty" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_PRESENCE_PENALTY,
+                            optional_params.get("presence_penalty"),
+                        )
+                    if "top_p" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_TOP_P, optional_params.get("top_p")
+                        )
+                    if "tools" in optional_params or "functions" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_REQUEST_FUNCTIONS,
+                            optional_params.get(
+                                "tools", optional_params.get("functions")
+                            ),
+                        )
+                    if "user" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_USER, optional_params.get("user")
+                        )
+                    if "max_tokens" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_REQUEST_MAX_TOKENS,
+                            kwargs.get("max_tokens"),
+                        )
+                    if "temperature" in optional_params:
+                        span.set_attribute(
+                            SpanAttributes.LLM_TEMPERATURE, kwargs.get("temperature")
+                        )
+
+                    for idx, prompt in enumerate(kwargs.get("messages")):
+                        span.set_attribute(
+                            f"{SpanAttributes.LLM_PROMPTS}.{idx}.role",
+                            prompt.get("role"),
+                        )
+                        span.set_attribute(
+                            f"{SpanAttributes.LLM_PROMPTS}.{idx}.content",
+                            prompt.get("content"),
+                        )
+
+                    span.set_attribute(
+                        SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model")
+                    )
+                    usage = response_obj.get("usage")
+                    if usage:
+                        span.set_attribute(
+                            SpanAttributes.LLM_USAGE_TOTAL_TOKENS,
+                            usage.get("total_tokens"),
+                        )
+                        span.set_attribute(
+                            SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
+                            usage.get("completion_tokens"),
+                        )
+                        span.set_attribute(
+                            SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
+                            usage.get("prompt_tokens"),
+                        )
+
+                    for idx, choice in enumerate(response_obj.get("choices")):
+                        span.set_attribute(
+                            f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason",
+                            choice.get("finish_reason"),
+                        )
+                        span.set_attribute(
+                            f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role",
+                            choice.get("message").get("role"),
+                        )
+                        span.set_attribute(
+                            f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content",
+                            choice.get("message").get("content"),
+                        )
+
+        except Exception as e:
+            print_verbose(f"Traceloop Layer Error - {e}")

litellm/integrations/weights_biases.py

@@ -0,0 +1,223 @@
+imported_openAIResponse = True
+try:
+    import io
+    import logging
+    import sys
+    from typing import Any, Dict, List, Optional, TypeVar
+
+    from wandb.sdk.data_types import trace_tree
+
+    if sys.version_info >= (3, 8):
+        from typing import Literal, Protocol
+    else:
+        from typing_extensions import Literal, Protocol
+
+    logger = logging.getLogger(__name__)
+
+    K = TypeVar("K", bound=str)
+    V = TypeVar("V")
+
+    class OpenAIResponse(Protocol[K, V]):  # type: ignore
+        # contains a (known) object attribute
+        object: Literal["chat.completion", "edit", "text_completion"]
+
+        def __getitem__(self, key: K) -> V:
+            ...  # pragma: no cover
+
+        def get(self, key: K, default: Optional[V] = None) -> Optional[V]:
+            ...  # pragma: no cover
+
+    class OpenAIRequestResponseResolver:
+        def __call__(
+            self,
+            request: Dict[str, Any],
+            response: OpenAIResponse,
+            time_elapsed: float,
+        ) -> Optional[trace_tree.WBTraceTree]:
+            try:
+                if response["object"] == "edit":
+                    return self._resolve_edit(request, response, time_elapsed)
+                elif response["object"] == "text_completion":
+                    return self._resolve_completion(request, response, time_elapsed)
+                elif response["object"] == "chat.completion":
+                    return self._resolve_chat_completion(
+                        request, response, time_elapsed
+                    )
+                else:
+                    logger.info(f"Unknown OpenAI response object: {response['object']}")
+            except Exception as e:
+                logger.warning(f"Failed to resolve request/response: {e}")
+            return None
+
+        @staticmethod
+        def results_to_trace_tree(
+            request: Dict[str, Any],
+            response: OpenAIResponse,
+            results: List[trace_tree.Result],
+            time_elapsed: float,
+        ) -> trace_tree.WBTraceTree:
+            """Converts the request, response, and results into a trace tree.
+
+            params:
+                request: The request dictionary
+                response: The response object
+                results: A list of results object
+                time_elapsed: The time elapsed in seconds
+            returns:
+                A wandb trace tree object.
+            """
+            start_time_ms = int(round(response["created"] * 1000))
+            end_time_ms = start_time_ms + int(round(time_elapsed * 1000))
+            span = trace_tree.Span(
+                name=f"{response.get('model', 'openai')}_{response['object']}_{response.get('created')}",
+                attributes=dict(response),  # type: ignore
+                start_time_ms=start_time_ms,
+                end_time_ms=end_time_ms,
+                span_kind=trace_tree.SpanKind.LLM,
+                results=results,
+            )
+            model_obj = {"request": request, "response": response, "_kind": "openai"}
+            return trace_tree.WBTraceTree(root_span=span, model_dict=model_obj)
+
+        def _resolve_edit(
+            self,
+            request: Dict[str, Any],
+            response: OpenAIResponse,
+            time_elapsed: float,
+        ) -> trace_tree.WBTraceTree:
+            """Resolves the request and response objects for `openai.Edit`."""
+            request_str = (
+                f"\n\n**Instruction**: {request['instruction']}\n\n"
+                f"**Input**: {request['input']}\n"
+            )
+            choices = [
+                f"\n\n**Edited**: {choice['text']}\n" for choice in response["choices"]
+            ]
+
+            return self._request_response_result_to_trace(
+                request=request,
+                response=response,
+                request_str=request_str,
+                choices=choices,
+                time_elapsed=time_elapsed,
+            )
+
+        def _resolve_completion(
+            self,
+            request: Dict[str, Any],
+            response: OpenAIResponse,
+            time_elapsed: float,
+        ) -> trace_tree.WBTraceTree:
+            """Resolves the request and response objects for `openai.Completion`."""
+            request_str = f"\n\n**Prompt**: {request['prompt']}\n"
+            choices = [
+                f"\n\n**Completion**: {choice['text']}\n"
+                for choice in response["choices"]
+            ]
+
+            return self._request_response_result_to_trace(
+                request=request,
+                response=response,
+                request_str=request_str,
+                choices=choices,
+                time_elapsed=time_elapsed,
+            )
+
+        def _resolve_chat_completion(
+            self,
+            request: Dict[str, Any],
+            response: OpenAIResponse,
+            time_elapsed: float,
+        ) -> trace_tree.WBTraceTree:
+            """Resolves the request and response objects for `openai.Completion`."""
+            prompt = io.StringIO()
+            for message in request["messages"]:
+                prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
+            request_str = prompt.getvalue()
+
+            choices = [
+                f"\n\n**{choice['message']['role']}**: {choice['message']['content']}\n"
+                for choice in response["choices"]
+            ]
+
+            return self._request_response_result_to_trace(
+                request=request,
+                response=response,
+                request_str=request_str,
+                choices=choices,
+                time_elapsed=time_elapsed,
+            )
+
+        def _request_response_result_to_trace(
+            self,
+            request: Dict[str, Any],
+            response: OpenAIResponse,
+            request_str: str,
+            choices: List[str],
+            time_elapsed: float,
+        ) -> trace_tree.WBTraceTree:
+            """Resolves the request and response objects for `openai.Completion`."""
+            results = [
+                trace_tree.Result(
+                    inputs={"request": request_str},
+                    outputs={"response": choice},
+                )
+                for choice in choices
+            ]
+            trace = self.results_to_trace_tree(request, response, results, time_elapsed)
+            return trace
+
+except:
+    imported_openAIResponse = False
+
+
+#### What this does ####
+#    On success, logs events to Langfuse
+import dotenv, os
+import requests
+import requests
+from datetime import datetime
+
+dotenv.load_dotenv()  # Loading env variables using dotenv
+import traceback
+
+
+class WeightsBiasesLogger:
+    # Class variables or attributes
+    def __init__(self):
+        try:
+            import wandb
+        except:
+            raise Exception(
+                "\033[91m wandb not installed, try running 'pip install wandb' to fix this error\033[0m"
+            )
+        if imported_openAIResponse == False:
+            raise Exception(
+                "\033[91m wandb not installed, try running 'pip install wandb' to fix this error\033[0m"
+            )
+        self.resolver = OpenAIRequestResponseResolver()
+
+    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+        # Method definition
+        import wandb
+
+        try:
+            print_verbose(f"W&B Logging - Enters logging function for model {kwargs}")
+            run = wandb.init()
+            print_verbose(response_obj)
+
+            trace = self.resolver(
+                kwargs, response_obj, (end_time - start_time).total_seconds()
+            )
+
+            if trace is not None:
+                run.log({"trace": trace})
+
+            run.finish()
+            print_verbose(
+                f"W&B Logging Logging - final response object: {response_obj}"
+            )
+        except:
+            # traceback.print_exc()
+            print_verbose(f"W&B Logging Layer Error - {traceback.format_exc()}")
+            pass

litellm/llms/__init__.py

@@ -0,0 +1 @@
+from . import *

litellm/llms/ai21.py

@@ -0,0 +1,212 @@
+import os, types, traceback
+import json
+from enum import Enum
+import requests
+import time, httpx
+from typing import Callable, Optional
+from litellm.utils import ModelResponse, Choices, Message
+import litellm
+
+
+class AI21Error(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(
+            method="POST", url="https://api.ai21.com/studio/v1/"
+        )
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class AI21Config:
+    """
+    Reference: https://docs.ai21.com/reference/j2-complete-ref
+
+    The class `AI21Config` provides configuration for the AI21's API interface. Below are the parameters:
+
+    - `numResults` (int32): Number of completions to sample and return. Optional, default is 1. If the temperature is greater than 0 (non-greedy decoding), a value greater than 1 can be meaningful.
+
+    - `maxTokens` (int32): The maximum number of tokens to generate per result. Optional, default is 16. If no `stopSequences` are given, generation stops after producing `maxTokens`.
+
+    - `minTokens` (int32): The minimum number of tokens to generate per result. Optional, default is 0. If `stopSequences` are given, they are ignored until `minTokens` are generated.
+
+    - `temperature` (float): Modifies the distribution from which tokens are sampled. Optional, default is 0.7. A value of 0 essentially disables sampling and results in greedy decoding.
+
+    - `topP` (float): Used for sampling tokens from the corresponding top percentile of probability mass. Optional, default is 1. For instance, a value of 0.9 considers only tokens comprising the top 90% probability mass.
+
+    - `stopSequences` (array of strings): Stops decoding if any of the input strings is generated. Optional.
+
+    - `topKReturn` (int32): Range between 0 to 10, including both. Optional, default is 0. Specifies the top-K alternative tokens to return. A non-zero value includes the string representations and log-probabilities for each of the top-K alternatives at each position.
+
+    - `frequencyPenalty` (object): Placeholder for frequency penalty object.
+
+    - `presencePenalty` (object): Placeholder for presence penalty object.
+
+    - `countPenalty` (object): Placeholder for count penalty object.
+    """
+
+    numResults: Optional[int] = None
+    maxTokens: Optional[int] = None
+    minTokens: Optional[int] = None
+    temperature: Optional[float] = None
+    topP: Optional[float] = None
+    stopSequences: Optional[list] = None
+    topKReturn: Optional[int] = None
+    frequencePenalty: Optional[dict] = None
+    presencePenalty: Optional[dict] = None
+    countPenalty: Optional[dict] = None
+
+    def __init__(
+        self,
+        numResults: Optional[int] = None,
+        maxTokens: Optional[int] = None,
+        minTokens: Optional[int] = None,
+        temperature: Optional[float] = None,
+        topP: Optional[float] = None,
+        stopSequences: Optional[list] = None,
+        topKReturn: Optional[int] = None,
+        frequencePenalty: Optional[dict] = None,
+        presencePenalty: Optional[dict] = None,
+        countPenalty: Optional[dict] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+def validate_environment(api_key):
+    if api_key is None:
+        raise ValueError(
+            "Missing AI21 API Key - A call is being made to ai21 but no key is set either in the environment variables or via params"
+        )
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "Authorization": "Bearer " + api_key,
+    }
+    return headers
+
+
+def completion(
+    model: str,
+    messages: list,
+    api_base: str,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    api_key,
+    logging_obj,
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+):
+    headers = validate_environment(api_key)
+    model = model
+    prompt = ""
+    for message in messages:
+        if "role" in message:
+            if message["role"] == "user":
+                prompt += f"{message['content']}"
+            else:
+                prompt += f"{message['content']}"
+        else:
+            prompt += f"{message['content']}"
+
+    ## Load Config
+    config = litellm.AI21Config.get_config()
+    for k, v in config.items():
+        if (
+            k not in optional_params
+        ):  # completion(top_k=3) > ai21_config(top_k=3) <- allows for dynamic variables to be passed in
+            optional_params[k] = v
+
+    data = {
+        "prompt": prompt,
+        # "instruction": prompt, # some baseten models require the prompt to be passed in via the 'instruction' kwarg
+        **optional_params,
+    }
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=prompt,
+        api_key=api_key,
+        additional_args={"complete_input_dict": data},
+    )
+    ## COMPLETION CALL
+    response = requests.post(
+        api_base + model + "/complete", headers=headers, data=json.dumps(data)
+    )
+    if response.status_code != 200:
+        raise AI21Error(status_code=response.status_code, message=response.text)
+    if "stream" in optional_params and optional_params["stream"] == True:
+        return response.iter_lines()
+    else:
+        ## LOGGING
+        logging_obj.post_call(
+            input=prompt,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        ## RESPONSE OBJECT
+        completion_response = response.json()
+        try:
+            choices_list = []
+            for idx, item in enumerate(completion_response["completions"]):
+                if len(item["data"]["text"]) > 0:
+                    message_obj = Message(content=item["data"]["text"])
+                else:
+                    message_obj = Message(content=None)
+                choice_obj = Choices(
+                    finish_reason=item["finishReason"]["reason"],
+                    index=idx + 1,
+                    message=message_obj,
+                )
+                choices_list.append(choice_obj)
+            model_response["choices"] = choices_list
+        except Exception as e:
+            raise AI21Error(
+                message=traceback.format_exc(), status_code=response.status_code
+            )
+
+        ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
+        prompt_tokens = len(encoding.encode(prompt))
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"].get("content"))
+        )
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = model
+        model_response["usage"] = {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+        }
+        return model_response
+
+
+def embedding():
+    # logic for parsing in - calling - parsing out model embedding calls
+    pass

litellm/llms/aleph_alpha.py

@@ -0,0 +1,304 @@
+import os, types
+import json
+from enum import Enum
+import requests
+import time
+from typing import Callable, Optional
+import litellm
+from litellm.utils import ModelResponse, Choices, Message, Usage
+import httpx
+
+
+class AlephAlphaError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(
+            method="POST", url="https://api.aleph-alpha.com/complete"
+        )
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class AlephAlphaConfig:
+    """
+    Reference: https://docs.aleph-alpha.com/api/complete/
+
+    The `AlephAlphaConfig` class represents the configuration for the Aleph Alpha API. Here are the properties:
+
+    - `maximum_tokens` (integer, required): The maximum number of tokens to be generated by the completion. The sum of input tokens and maximum tokens may not exceed 2048.
+
+    - `minimum_tokens` (integer, optional; default value: 0): Generate at least this number of tokens before an end-of-text token is generated.
+
+    - `echo` (boolean, optional; default value: false): Whether to echo the prompt in the completion.
+
+    - `temperature` (number, nullable; default value: 0): Adjusts how creatively the model generates outputs. Use combinations of temperature, top_k, and top_p sensibly.
+
+    - `top_k` (integer, nullable; default value: 0): Introduces randomness into token generation by considering the top k most likely options.
+
+    - `top_p` (number, nullable; default value: 0): Adds randomness by considering the smallest set of tokens whose cumulative probability exceeds top_p.
+
+    - `presence_penalty`, `frequency_penalty`, `sequence_penalty` (number, nullable; default value: 0): Various penalties that can reduce repetition.
+
+    - `sequence_penalty_min_length` (integer; default value: 2): Minimum number of tokens to be considered as a sequence.
+
+    - `repetition_penalties_include_prompt`, `repetition_penalties_include_completion`, `use_multiplicative_presence_penalty`,`use_multiplicative_frequency_penalty`,`use_multiplicative_sequence_penalty` (boolean, nullable; default value: false): Various settings that adjust how the repetition penalties are applied.
+
+    - `penalty_bias` (string, nullable): Text used in addition to the penalized tokens for repetition penalties.
+
+    - `penalty_exceptions` (string[], nullable): Strings that may be generated without penalty.
+
+    - `penalty_exceptions_include_stop_sequences` (boolean, nullable; default value: true): Include all stop_sequences in penalty_exceptions.
+
+    - `best_of` (integer, nullable; default value: 1): The number of completions will be generated on the server side.
+
+    - `n` (integer, nullable; default value: 1): The number of completions to return.
+
+    - `logit_bias` (object, nullable): Adjust the logit scores before sampling.
+
+    - `log_probs` (integer, nullable): Number of top log probabilities for each token generated.
+
+    - `stop_sequences` (string[], nullable): List of strings that will stop generation if they're generated.
+
+    - `tokens` (boolean, nullable; default value: false): Flag indicating whether individual tokens of the completion should be returned or not.
+
+    - `raw_completion` (boolean; default value: false): if True, the raw completion of the model will be returned.
+
+    - `disable_optimizations` (boolean, nullable; default value: false): Disables any applied optimizations to both your prompt and completion.
+
+    - `completion_bias_inclusion`, `completion_bias_exclusion` (string[], default value: []): Set of strings to bias the generation of tokens.
+
+    - `completion_bias_inclusion_first_token_only`, `completion_bias_exclusion_first_token_only` (boolean; default value: false): Consider only the first token for the completion_bias_inclusion/exclusion.
+
+    - `contextual_control_threshold` (number, nullable): Control over how similar tokens are controlled.
+
+    - `control_log_additive` (boolean; default value: true): Method of applying control to attention scores.
+    """
+
+    maximum_tokens: Optional[
+        int
+    ] = litellm.max_tokens  # aleph alpha requires max tokens
+    minimum_tokens: Optional[int] = None
+    echo: Optional[bool] = None
+    temperature: Optional[int] = None
+    top_k: Optional[int] = None
+    top_p: Optional[int] = None
+    presence_penalty: Optional[int] = None
+    frequency_penalty: Optional[int] = None
+    sequence_penalty: Optional[int] = None
+    sequence_penalty_min_length: Optional[int] = None
+    repetition_penalties_include_prompt: Optional[bool] = None
+    repetition_penalties_include_completion: Optional[bool] = None
+    use_multiplicative_presence_penalty: Optional[bool] = None
+    use_multiplicative_frequency_penalty: Optional[bool] = None
+    use_multiplicative_sequence_penalty: Optional[bool] = None
+    penalty_bias: Optional[str] = None
+    penalty_exceptions_include_stop_sequences: Optional[bool] = None
+    best_of: Optional[int] = None
+    n: Optional[int] = None
+    logit_bias: Optional[dict] = None
+    log_probs: Optional[int] = None
+    stop_sequences: Optional[list] = None
+    tokens: Optional[bool] = None
+    raw_completion: Optional[bool] = None
+    disable_optimizations: Optional[bool] = None
+    completion_bias_inclusion: Optional[list] = None
+    completion_bias_exclusion: Optional[list] = None
+    completion_bias_inclusion_first_token_only: Optional[bool] = None
+    completion_bias_exclusion_first_token_only: Optional[bool] = None
+    contextual_control_threshold: Optional[int] = None
+    control_log_additive: Optional[bool] = None
+
+    def __init__(
+        self,
+        maximum_tokens: Optional[int] = None,
+        minimum_tokens: Optional[int] = None,
+        echo: Optional[bool] = None,
+        temperature: Optional[int] = None,
+        top_k: Optional[int] = None,
+        top_p: Optional[int] = None,
+        presence_penalty: Optional[int] = None,
+        frequency_penalty: Optional[int] = None,
+        sequence_penalty: Optional[int] = None,
+        sequence_penalty_min_length: Optional[int] = None,
+        repetition_penalties_include_prompt: Optional[bool] = None,
+        repetition_penalties_include_completion: Optional[bool] = None,
+        use_multiplicative_presence_penalty: Optional[bool] = None,
+        use_multiplicative_frequency_penalty: Optional[bool] = None,
+        use_multiplicative_sequence_penalty: Optional[bool] = None,
+        penalty_bias: Optional[str] = None,
+        penalty_exceptions_include_stop_sequences: Optional[bool] = None,
+        best_of: Optional[int] = None,
+        n: Optional[int] = None,
+        logit_bias: Optional[dict] = None,
+        log_probs: Optional[int] = None,
+        stop_sequences: Optional[list] = None,
+        tokens: Optional[bool] = None,
+        raw_completion: Optional[bool] = None,
+        disable_optimizations: Optional[bool] = None,
+        completion_bias_inclusion: Optional[list] = None,
+        completion_bias_exclusion: Optional[list] = None,
+        completion_bias_inclusion_first_token_only: Optional[bool] = None,
+        completion_bias_exclusion_first_token_only: Optional[bool] = None,
+        contextual_control_threshold: Optional[int] = None,
+        control_log_additive: Optional[bool] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+def validate_environment(api_key):
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+    }
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    return headers
+
+
+def completion(
+    model: str,
+    messages: list,
+    api_base: str,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    api_key,
+    logging_obj,
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+    default_max_tokens_to_sample=None,
+):
+    headers = validate_environment(api_key)
+
+    ## Load Config
+    config = litellm.AlephAlphaConfig.get_config()
+    for k, v in config.items():
+        if (
+            k not in optional_params
+        ):  # completion(top_k=3) > aleph_alpha_config(top_k=3) <- allows for dynamic variables to be passed in
+            optional_params[k] = v
+
+    completion_url = api_base
+    model = model
+    prompt = ""
+    if "control" in model:  # follow the ###Instruction / ###Response format
+        for idx, message in enumerate(messages):
+            if "role" in message:
+                if (
+                    idx == 0
+                ):  # set first message as instruction (required), let later user messages be input
+                    prompt += f"###Instruction: {message['content']}"
+                else:
+                    if message["role"] == "system":
+                        prompt += f"###Instruction: {message['content']}"
+                    elif message["role"] == "user":
+                        prompt += f"###Input: {message['content']}"
+                    else:
+                        prompt += f"###Response: {message['content']}"
+            else:
+                prompt += f"{message['content']}"
+    else:
+        prompt = " ".join(message["content"] for message in messages)
+    data = {
+        "model": model,
+        "prompt": prompt,
+        **optional_params,
+    }
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=prompt,
+        api_key=api_key,
+        additional_args={"complete_input_dict": data},
+    )
+    ## COMPLETION CALL
+    response = requests.post(
+        completion_url,
+        headers=headers,
+        data=json.dumps(data),
+        stream=optional_params["stream"] if "stream" in optional_params else False,
+    )
+    if "stream" in optional_params and optional_params["stream"] == True:
+        return response.iter_lines()
+    else:
+        ## LOGGING
+        logging_obj.post_call(
+            input=prompt,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response.text}")
+        ## RESPONSE OBJECT
+        completion_response = response.json()
+        if "error" in completion_response:
+            raise AlephAlphaError(
+                message=completion_response["error"],
+                status_code=response.status_code,
+            )
+        else:
+            try:
+                choices_list = []
+                for idx, item in enumerate(completion_response["completions"]):
+                    if len(item["completion"]) > 0:
+                        message_obj = Message(content=item["completion"])
+                    else:
+                        message_obj = Message(content=None)
+                    choice_obj = Choices(
+                        finish_reason=item["finish_reason"],
+                        index=idx + 1,
+                        message=message_obj,
+                    )
+                    choices_list.append(choice_obj)
+                model_response["choices"] = choices_list
+            except:
+                raise AlephAlphaError(
+                    message=json.dumps(completion_response),
+                    status_code=response.status_code,
+                )
+
+        ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
+        prompt_tokens = len(encoding.encode(prompt))
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"]["content"])
+        )
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        model_response.usage = usage
+        return model_response
+
+
+def embedding():
+    # logic for parsing in - calling - parsing out model embedding calls
+    pass

litellm/llms/anthropic.py

@@ -0,0 +1,215 @@
+import os, types
+import json
+from enum import Enum
+import requests
+import time
+from typing import Callable, Optional
+from litellm.utils import ModelResponse, Usage
+import litellm
+from .prompt_templates.factory import prompt_factory, custom_prompt
+import httpx
+
+
+class AnthropicConstants(Enum):
+    HUMAN_PROMPT = "\n\nHuman: "
+    AI_PROMPT = "\n\nAssistant: "
+
+
+class AnthropicError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(
+            method="POST", url="https://api.anthropic.com/v1/complete"
+        )
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class AnthropicConfig:
+    """
+    Reference: https://docs.anthropic.com/claude/reference/complete_post
+
+    to pass metadata to anthropic, it's {"user_id": "any-relevant-information"}
+    """
+
+    max_tokens_to_sample: Optional[
+        int
+    ] = litellm.max_tokens  # anthropic requires a default
+    stop_sequences: Optional[list] = None
+    temperature: Optional[int] = None
+    top_p: Optional[int] = None
+    top_k: Optional[int] = None
+    metadata: Optional[dict] = None
+
+    def __init__(
+        self,
+        max_tokens_to_sample: Optional[int] = 256,  # anthropic requires a default
+        stop_sequences: Optional[list] = None,
+        temperature: Optional[int] = None,
+        top_p: Optional[int] = None,
+        top_k: Optional[int] = None,
+        metadata: Optional[dict] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+# makes headers for API call
+def validate_environment(api_key):
+    if api_key is None:
+        raise ValueError(
+            "Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params"
+        )
+    headers = {
+        "accept": "application/json",
+        "anthropic-version": "2023-06-01",
+        "content-type": "application/json",
+        "x-api-key": api_key,
+    }
+    return headers
+
+
+def completion(
+    model: str,
+    messages: list,
+    api_base: str,
+    custom_prompt_dict: dict,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    api_key,
+    logging_obj,
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+):
+    headers = validate_environment(api_key)
+    if model in custom_prompt_dict:
+        # check if the model has a registered custom prompt
+        model_prompt_details = custom_prompt_dict[model]
+        prompt = custom_prompt(
+            role_dict=model_prompt_details["roles"],
+            initial_prompt_value=model_prompt_details["initial_prompt_value"],
+            final_prompt_value=model_prompt_details["final_prompt_value"],
+            messages=messages,
+        )
+    else:
+        prompt = prompt_factory(
+            model=model, messages=messages, custom_llm_provider="anthropic"
+        )
+
+    ## Load Config
+    config = litellm.AnthropicConfig.get_config()
+    for k, v in config.items():
+        if (
+            k not in optional_params
+        ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
+            optional_params[k] = v
+
+    data = {
+        "model": model,
+        "prompt": prompt,
+        **optional_params,
+    }
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=prompt,
+        api_key=api_key,
+        additional_args={"complete_input_dict": data, "api_base": api_base},
+    )
+
+    ## COMPLETION CALL
+    if "stream" in optional_params and optional_params["stream"] == True:
+        response = requests.post(
+            api_base,
+            headers=headers,
+            data=json.dumps(data),
+            stream=optional_params["stream"],
+        )
+
+        if response.status_code != 200:
+            raise AnthropicError(
+                status_code=response.status_code, message=response.text
+            )
+
+        return response.iter_lines()
+    else:
+        response = requests.post(api_base, headers=headers, data=json.dumps(data))
+        if response.status_code != 200:
+            raise AnthropicError(
+                status_code=response.status_code, message=response.text
+            )
+
+        ## LOGGING
+        logging_obj.post_call(
+            input=prompt,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response.text}")
+        ## RESPONSE OBJECT
+        try:
+            completion_response = response.json()
+        except:
+            raise AnthropicError(
+                message=response.text, status_code=response.status_code
+            )
+        if "error" in completion_response:
+            raise AnthropicError(
+                message=str(completion_response["error"]),
+                status_code=response.status_code,
+            )
+        else:
+            if len(completion_response["completion"]) > 0:
+                model_response["choices"][0]["message"][
+                    "content"
+                ] = completion_response["completion"]
+            model_response.choices[0].finish_reason = completion_response["stop_reason"]
+
+        ## CALCULATING USAGE
+        prompt_tokens = len(
+            encoding.encode(prompt)
+        )  ##[TODO] use the anthropic tokenizer here
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+        )  ##[TODO] use the anthropic tokenizer here
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        model_response.usage = usage
+        return model_response
+
+
+def embedding():
+    # logic for parsing in - calling - parsing out model embedding calls
+    pass

litellm/llms/azure.py

@@ -0,0 +1,799 @@
+from typing import Optional, Union, Any
+import types, requests
+from .base import BaseLLM
+from litellm.utils import (
+    ModelResponse,
+    Choices,
+    Message,
+    CustomStreamWrapper,
+    convert_to_model_response_object,
+)
+from typing import Callable, Optional
+from litellm import OpenAIConfig
+import litellm, json
+import httpx
+from .custom_httpx.azure_dall_e_2 import CustomHTTPTransport, AsyncCustomHTTPTransport
+from openai import AzureOpenAI, AsyncAzureOpenAI
+
+
+class AzureOpenAIError(Exception):
+    def __init__(
+        self,
+        status_code,
+        message,
+        request: Optional[httpx.Request] = None,
+        response: Optional[httpx.Response] = None,
+    ):
+        self.status_code = status_code
+        self.message = message
+        if request:
+            self.request = request
+        else:
+            self.request = httpx.Request(method="POST", url="https://api.openai.com/v1")
+        if response:
+            self.response = response
+        else:
+            self.response = httpx.Response(
+                status_code=status_code, request=self.request
+            )
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class AzureOpenAIConfig(OpenAIConfig):
+    """
+    Reference: https://platform.openai.com/docs/api-reference/chat/create
+
+    The class `AzureOpenAIConfig` provides configuration for the OpenAI's Chat API interface, for use with Azure. It inherits from `OpenAIConfig`. Below are the parameters::
+
+    - `frequency_penalty` (number or null): Defaults to 0. Allows a value between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, thereby minimizing repetition.
+
+    - `function_call` (string or object): This optional parameter controls how the model calls functions.
+
+    - `functions` (array): An optional parameter. It is a list of functions for which the model may generate JSON inputs.
+
+    - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion.
+
+    - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion.
+
+    - `n` (integer or null): This optional parameter helps to set how many chat completion choices to generate for each input message.
+
+    - `presence_penalty` (number or null): Defaults to 0. It penalizes new tokens based on if they appear in the text so far, hence increasing the model's likelihood to talk about new topics.
+
+    - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens.
+
+    - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2.
+
+    - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
+    """
+
+    def __init__(
+        self,
+        frequency_penalty: Optional[int] = None,
+        function_call: Optional[Union[str, dict]] = None,
+        functions: Optional[list] = None,
+        logit_bias: Optional[dict] = None,
+        max_tokens: Optional[int] = None,
+        n: Optional[int] = None,
+        presence_penalty: Optional[int] = None,
+        stop: Optional[Union[str, list]] = None,
+        temperature: Optional[int] = None,
+        top_p: Optional[int] = None,
+    ) -> None:
+        super().__init__(
+            frequency_penalty,
+            function_call,
+            functions,
+            logit_bias,
+            max_tokens,
+            n,
+            presence_penalty,
+            stop,
+            temperature,
+            top_p,
+        )
+
+
+class AzureChatCompletion(BaseLLM):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def validate_environment(self, api_key, azure_ad_token):
+        headers = {
+            "content-type": "application/json",
+        }
+        if api_key is not None:
+            headers["api-key"] = api_key
+        elif azure_ad_token is not None:
+            headers["Authorization"] = f"Bearer {azure_ad_token}"
+        return headers
+
+    def completion(
+        self,
+        model: str,
+        messages: list,
+        model_response: ModelResponse,
+        api_key: str,
+        api_base: str,
+        api_version: str,
+        api_type: str,
+        azure_ad_token: str,
+        print_verbose: Callable,
+        timeout,
+        logging_obj,
+        optional_params,
+        litellm_params,
+        logger_fn,
+        acompletion: bool = False,
+        headers: Optional[dict] = None,
+        client=None,
+    ):
+        super().completion()
+        exception_mapping_worked = False
+        try:
+            if model is None or messages is None:
+                raise AzureOpenAIError(
+                    status_code=422, message=f"Missing model or messages"
+                )
+
+            max_retries = optional_params.pop("max_retries", 2)
+
+            ### CHECK IF CLOUDFLARE AI GATEWAY ###
+            ### if so - set the model as part of the base url
+            if "gateway.ai.cloudflare.com" in api_base:
+                ## build base url - assume api base includes resource name
+                if client is None:
+                    if not api_base.endswith("/"):
+                        api_base += "/"
+                    api_base += f"{model}"
+
+                    azure_client_params = {
+                        "api_version": api_version,
+                        "base_url": f"{api_base}",
+                        "http_client": litellm.client_session,
+                        "max_retries": max_retries,
+                        "timeout": timeout,
+                    }
+                    if api_key is not None:
+                        azure_client_params["api_key"] = api_key
+                    elif azure_ad_token is not None:
+                        azure_client_params["azure_ad_token"] = azure_ad_token
+
+                    if acompletion is True:
+                        client = AsyncAzureOpenAI(**azure_client_params)
+                    else:
+                        client = AzureOpenAI(**azure_client_params)
+
+                data = {"model": None, "messages": messages, **optional_params}
+            else:
+                data = {
+                    "model": model,  # type: ignore
+                    "messages": messages,
+                    **optional_params,
+                }
+
+            if acompletion is True:
+                if optional_params.get("stream", False):
+                    return self.async_streaming(
+                        logging_obj=logging_obj,
+                        api_base=api_base,
+                        data=data,
+                        model=model,
+                        api_key=api_key,
+                        api_version=api_version,
+                        azure_ad_token=azure_ad_token,
+                        timeout=timeout,
+                        client=client,
+                    )
+                else:
+                    return self.acompletion(
+                        api_base=api_base,
+                        data=data,
+                        model_response=model_response,
+                        api_key=api_key,
+                        api_version=api_version,
+                        model=model,
+                        azure_ad_token=azure_ad_token,
+                        timeout=timeout,
+                        client=client,
+                        logging_obj=logging_obj,
+                    )
+            elif "stream" in optional_params and optional_params["stream"] == True:
+                return self.streaming(
+                    logging_obj=logging_obj,
+                    api_base=api_base,
+                    data=data,
+                    model=model,
+                    api_key=api_key,
+                    api_version=api_version,
+                    azure_ad_token=azure_ad_token,
+                    timeout=timeout,
+                    client=client,
+                )
+            else:
+                ## LOGGING
+                logging_obj.pre_call(
+                    input=messages,
+                    api_key=api_key,
+                    additional_args={
+                        "headers": {
+                            "api_key": api_key,
+                            "azure_ad_token": azure_ad_token,
+                        },
+                        "api_version": api_version,
+                        "api_base": api_base,
+                        "complete_input_dict": data,
+                    },
+                )
+                if not isinstance(max_retries, int):
+                    raise AzureOpenAIError(
+                        status_code=422, message="max retries must be an int"
+                    )
+                # init AzureOpenAI Client
+                azure_client_params = {
+                    "api_version": api_version,
+                    "azure_endpoint": api_base,
+                    "azure_deployment": model,
+                    "http_client": litellm.client_session,
+                    "max_retries": max_retries,
+                    "timeout": timeout,
+                }
+                if api_key is not None:
+                    azure_client_params["api_key"] = api_key
+                elif azure_ad_token is not None:
+                    azure_client_params["azure_ad_token"] = azure_ad_token
+                if client is None:
+                    azure_client = AzureOpenAI(**azure_client_params)
+                else:
+                    azure_client = client
+                response = azure_client.chat.completions.create(**data, timeout=timeout)  # type: ignore
+                stringified_response = response.model_dump()
+                ## LOGGING
+                logging_obj.post_call(
+                    input=messages,
+                    api_key=api_key,
+                    original_response=stringified_response,
+                    additional_args={
+                        "headers": headers,
+                        "api_version": api_version,
+                        "api_base": api_base,
+                    },
+                )
+                return convert_to_model_response_object(
+                    response_object=stringified_response,
+                    model_response_object=model_response,
+                )
+        except AzureOpenAIError as e:
+            exception_mapping_worked = True
+            raise e
+        except Exception as e:
+            if hasattr(e, "status_code"):
+                raise AzureOpenAIError(status_code=e.status_code, message=str(e))
+            else:
+                raise AzureOpenAIError(status_code=500, message=str(e))
+
+    async def acompletion(
+        self,
+        api_key: str,
+        api_version: str,
+        model: str,
+        api_base: str,
+        data: dict,
+        timeout: Any,
+        model_response: ModelResponse,
+        azure_ad_token: Optional[str] = None,
+        client=None,  # this is the AsyncAzureOpenAI
+        logging_obj=None,
+    ):
+        response = None
+        try:
+            max_retries = data.pop("max_retries", 2)
+            if not isinstance(max_retries, int):
+                raise AzureOpenAIError(
+                    status_code=422, message="max retries must be an int"
+                )
+
+            # init AzureOpenAI Client
+            azure_client_params = {
+                "api_version": api_version,
+                "azure_endpoint": api_base,
+                "azure_deployment": model,
+                "http_client": litellm.client_session,
+                "max_retries": max_retries,
+                "timeout": timeout,
+            }
+            if api_key is not None:
+                azure_client_params["api_key"] = api_key
+            elif azure_ad_token is not None:
+                azure_client_params["azure_ad_token"] = azure_ad_token
+            if client is None:
+                azure_client = AsyncAzureOpenAI(**azure_client_params)
+            else:
+                azure_client = client
+            ## LOGGING
+            logging_obj.pre_call(
+                input=data["messages"],
+                api_key=azure_client.api_key,
+                additional_args={
+                    "headers": {"Authorization": f"Bearer {azure_client.api_key}"},
+                    "api_base": azure_client._base_url._uri_reference,
+                    "acompletion": True,
+                    "complete_input_dict": data,
+                },
+            )
+            response = await azure_client.chat.completions.create(
+                **data, timeout=timeout
+            )
+            return convert_to_model_response_object(
+                response_object=response.model_dump(),
+                model_response_object=model_response,
+            )
+        except AzureOpenAIError as e:
+            exception_mapping_worked = True
+            raise e
+        except Exception as e:
+            if hasattr(e, "status_code"):
+                raise e
+            else:
+                raise AzureOpenAIError(status_code=500, message=str(e))
+
+    def streaming(
+        self,
+        logging_obj,
+        api_base: str,
+        api_key: str,
+        api_version: str,
+        data: dict,
+        model: str,
+        timeout: Any,
+        azure_ad_token: Optional[str] = None,
+        client=None,
+    ):
+        max_retries = data.pop("max_retries", 2)
+        if not isinstance(max_retries, int):
+            raise AzureOpenAIError(
+                status_code=422, message="max retries must be an int"
+            )
+        # init AzureOpenAI Client
+        azure_client_params = {
+            "api_version": api_version,
+            "azure_endpoint": api_base,
+            "azure_deployment": model,
+            "http_client": litellm.client_session,
+            "max_retries": max_retries,
+            "timeout": timeout,
+        }
+        if api_key is not None:
+            azure_client_params["api_key"] = api_key
+        elif azure_ad_token is not None:
+            azure_client_params["azure_ad_token"] = azure_ad_token
+        if client is None:
+            azure_client = AzureOpenAI(**azure_client_params)
+        else:
+            azure_client = client
+        ## LOGGING
+        logging_obj.pre_call(
+            input=data["messages"],
+            api_key=azure_client.api_key,
+            additional_args={
+                "headers": {"Authorization": f"Bearer {azure_client.api_key}"},
+                "api_base": azure_client._base_url._uri_reference,
+                "acompletion": True,
+                "complete_input_dict": data,
+            },
+        )
+        response = azure_client.chat.completions.create(**data, timeout=timeout)
+        streamwrapper = CustomStreamWrapper(
+            completion_stream=response,
+            model=model,
+            custom_llm_provider="azure",
+            logging_obj=logging_obj,
+        )
+        return streamwrapper
+
+    async def async_streaming(
+        self,
+        logging_obj,
+        api_base: str,
+        api_key: str,
+        api_version: str,
+        data: dict,
+        model: str,
+        timeout: Any,
+        azure_ad_token: Optional[str] = None,
+        client=None,
+    ):
+        try:
+            # init AzureOpenAI Client
+            azure_client_params = {
+                "api_version": api_version,
+                "azure_endpoint": api_base,
+                "azure_deployment": model,
+                "http_client": litellm.client_session,
+                "max_retries": data.pop("max_retries", 2),
+                "timeout": timeout,
+            }
+            if api_key is not None:
+                azure_client_params["api_key"] = api_key
+            elif azure_ad_token is not None:
+                azure_client_params["azure_ad_token"] = azure_ad_token
+            if client is None:
+                azure_client = AsyncAzureOpenAI(**azure_client_params)
+            else:
+                azure_client = client
+            ## LOGGING
+            logging_obj.pre_call(
+                input=data["messages"],
+                api_key=azure_client.api_key,
+                additional_args={
+                    "headers": {"Authorization": f"Bearer {azure_client.api_key}"},
+                    "api_base": azure_client._base_url._uri_reference,
+                    "acompletion": True,
+                    "complete_input_dict": data,
+                },
+            )
+            response = await azure_client.chat.completions.create(
+                **data, timeout=timeout
+            )
+            # return response
+            streamwrapper = CustomStreamWrapper(
+                completion_stream=response,
+                model=model,
+                custom_llm_provider="azure",
+                logging_obj=logging_obj,
+            )
+            return streamwrapper  ## DO NOT make this into an async for ... loop, it will yield an async generator, which won't raise errors if the response fails
+        except Exception as e:
+            if hasattr(e, "status_code"):
+                raise AzureOpenAIError(status_code=e.status_code, message=str(e))
+            else:
+                raise AzureOpenAIError(status_code=500, message=str(e))
+
+    async def aembedding(
+        self,
+        data: dict,
+        model_response: ModelResponse,
+        azure_client_params: dict,
+        api_key: str,
+        input: list,
+        client=None,
+        logging_obj=None,
+        timeout=None,
+    ):
+        response = None
+        try:
+            if client is None:
+                openai_aclient = AsyncAzureOpenAI(**azure_client_params)
+            else:
+                openai_aclient = client
+            response = await openai_aclient.embeddings.create(**data, timeout=timeout)
+            stringified_response = response.model_dump()
+            ## LOGGING
+            logging_obj.post_call(
+                input=input,
+                api_key=api_key,
+                additional_args={"complete_input_dict": data},
+                original_response=stringified_response,
+            )
+            return convert_to_model_response_object(
+                response_object=stringified_response,
+                model_response_object=model_response,
+                response_type="embedding",
+            )
+        except Exception as e:
+            ## LOGGING
+            logging_obj.post_call(
+                input=input,
+                api_key=api_key,
+                additional_args={"complete_input_dict": data},
+                original_response=str(e),
+            )
+            raise e
+
+    def embedding(
+        self,
+        model: str,
+        input: list,
+        api_key: str,
+        api_base: str,
+        api_version: str,
+        timeout: float,
+        logging_obj=None,
+        model_response=None,
+        optional_params=None,
+        azure_ad_token: Optional[str] = None,
+        client=None,
+        aembedding=None,
+    ):
+        super().embedding()
+        exception_mapping_worked = False
+        if self._client_session is None:
+            self._client_session = self.create_client_session()
+        try:
+            data = {"model": model, "input": input, **optional_params}
+            max_retries = data.pop("max_retries", 2)
+            if not isinstance(max_retries, int):
+                raise AzureOpenAIError(
+                    status_code=422, message="max retries must be an int"
+                )
+
+            # init AzureOpenAI Client
+            azure_client_params = {
+                "api_version": api_version,
+                "azure_endpoint": api_base,
+                "azure_deployment": model,
+                "http_client": litellm.client_session,
+                "max_retries": max_retries,
+                "timeout": timeout,
+            }
+            if api_key is not None:
+                azure_client_params["api_key"] = api_key
+            elif azure_ad_token is not None:
+                azure_client_params["azure_ad_token"] = azure_ad_token
+
+            ## LOGGING
+            logging_obj.pre_call(
+                input=input,
+                api_key=api_key,
+                additional_args={
+                    "complete_input_dict": data,
+                    "headers": {"api_key": api_key, "azure_ad_token": azure_ad_token},
+                },
+            )
+
+            if aembedding == True:
+                response = self.aembedding(
+                    data=data,
+                    input=input,
+                    logging_obj=logging_obj,
+                    api_key=api_key,
+                    model_response=model_response,
+                    azure_client_params=azure_client_params,
+                    timeout=timeout,
+                )
+                return response
+            if client is None:
+                azure_client = AzureOpenAI(**azure_client_params)  # type: ignore
+            else:
+                azure_client = client
+            ## COMPLETION CALL
+            response = azure_client.embeddings.create(**data, timeout=timeout)  # type: ignore
+            ## LOGGING
+            logging_obj.post_call(
+                input=input,
+                api_key=api_key,
+                additional_args={"complete_input_dict": data, "api_base": api_base},
+                original_response=response,
+            )
+
+            return convert_to_model_response_object(response_object=response.model_dump(), model_response_object=model_response, response_type="embedding")  # type: ignore
+        except AzureOpenAIError as e:
+            exception_mapping_worked = True
+            raise e
+        except Exception as e:
+            if hasattr(e, "status_code"):
+                raise AzureOpenAIError(status_code=e.status_code, message=str(e))
+            else:
+                raise AzureOpenAIError(status_code=500, message=str(e))
+
+    async def aimage_generation(
+        self,
+        data: dict,
+        model_response: ModelResponse,
+        azure_client_params: dict,
+        api_key: str,
+        input: list,
+        client=None,
+        logging_obj=None,
+        timeout=None,
+    ):
+        response = None
+        try:
+            if client is None:
+                client_session = litellm.aclient_session or httpx.AsyncClient(
+                    transport=AsyncCustomHTTPTransport(),
+                )
+                openai_aclient = AsyncAzureOpenAI(
+                    http_client=client_session, **azure_client_params
+                )
+            else:
+                openai_aclient = client
+            response = await openai_aclient.images.generate(**data, timeout=timeout)
+            stringified_response = response.model_dump()
+            ## LOGGING
+            logging_obj.post_call(
+                input=input,
+                api_key=api_key,
+                additional_args={"complete_input_dict": data},
+                original_response=stringified_response,
+            )
+            return convert_to_model_response_object(
+                response_object=stringified_response,
+                model_response_object=model_response,
+                response_type="image_generation",
+            )
+        except Exception as e:
+            ## LOGGING
+            logging_obj.post_call(
+                input=input,
+                api_key=api_key,
+                additional_args={"complete_input_dict": data},
+                original_response=str(e),
+            )
+            raise e
+
+    def image_generation(
+        self,
+        prompt: str,
+        timeout: float,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        api_version: Optional[str] = None,
+        model_response: Optional[litellm.utils.ImageResponse] = None,
+        azure_ad_token: Optional[str] = None,
+        logging_obj=None,
+        optional_params=None,
+        client=None,
+        aimg_generation=None,
+    ):
+        exception_mapping_worked = False
+        try:
+            if model and len(model) > 0:
+                model = model
+            else:
+                model = None
+            data = {"model": model, "prompt": prompt, **optional_params}
+            max_retries = data.pop("max_retries", 2)
+            if not isinstance(max_retries, int):
+                raise AzureOpenAIError(
+                    status_code=422, message="max retries must be an int"
+                )
+
+            # init AzureOpenAI Client
+            azure_client_params = {
+                "api_version": api_version,
+                "azure_endpoint": api_base,
+                "azure_deployment": model,
+                "max_retries": max_retries,
+                "timeout": timeout,
+            }
+            if api_key is not None:
+                azure_client_params["api_key"] = api_key
+            elif azure_ad_token is not None:
+                azure_client_params["azure_ad_token"] = azure_ad_token
+
+            if aimg_generation == True:
+                response = self.aimage_generation(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_key=api_key, client=client, azure_client_params=azure_client_params, timeout=timeout)  # type: ignore
+                return response
+
+            if client is None:
+                client_session = litellm.client_session or httpx.Client(
+                    transport=CustomHTTPTransport(),
+                )
+                azure_client = AzureOpenAI(http_client=client_session, **azure_client_params)  # type: ignore
+            else:
+                azure_client = client
+
+            ## LOGGING
+            logging_obj.pre_call(
+                input=prompt,
+                api_key=azure_client.api_key,
+                additional_args={
+                    "headers": {"Authorization": f"Bearer {azure_client.api_key}"},
+                    "api_base": azure_client._base_url._uri_reference,
+                    "acompletion": False,
+                    "complete_input_dict": data,
+                },
+            )
+
+            ## COMPLETION CALL
+            response = azure_client.images.generate(**data, timeout=timeout)  # type: ignore
+            ## LOGGING
+            logging_obj.post_call(
+                input=input,
+                api_key=api_key,
+                additional_args={"complete_input_dict": data},
+                original_response=response,
+            )
+            # return response
+            return convert_to_model_response_object(response_object=response.model_dump(), model_response_object=model_response, response_type="image_generation")  # type: ignore
+        except AzureOpenAIError as e:
+            exception_mapping_worked = True
+            raise e
+        except Exception as e:
+            if hasattr(e, "status_code"):
+                raise AzureOpenAIError(status_code=e.status_code, message=str(e))
+            else:
+                raise AzureOpenAIError(status_code=500, message=str(e))
+
+    async def ahealth_check(
+        self,
+        model: Optional[str],
+        api_key: str,
+        api_base: str,
+        api_version: str,
+        timeout: float,
+        mode: str,
+        messages: Optional[list] = None,
+        input: Optional[list] = None,
+        prompt: Optional[str] = None,
+    ):
+        client_session = litellm.aclient_session or httpx.AsyncClient(
+            transport=AsyncCustomHTTPTransport(),  # handle dall-e-2 calls
+        )
+        if "gateway.ai.cloudflare.com" in api_base:
+            ## build base url - assume api base includes resource name
+            if not api_base.endswith("/"):
+                api_base += "/"
+            api_base += f"{model}"
+            client = AsyncAzureOpenAI(
+                base_url=api_base,
+                api_version=api_version,
+                api_key=api_key,
+                timeout=timeout,
+                http_client=client_session,
+            )
+            model = None
+            # cloudflare ai gateway, needs model=None
+        else:
+            client = AsyncAzureOpenAI(
+                api_version=api_version,
+                azure_endpoint=api_base,
+                api_key=api_key,
+                timeout=timeout,
+                http_client=client_session,
+            )
+
+            # only run this check if it's not cloudflare ai gateway
+            if model is None and mode != "image_generation":
+                raise Exception("model is not set")
+
+        completion = None
+
+        if mode == "completion":
+            completion = await client.completions.with_raw_response.create(
+                model=model,  # type: ignore
+                prompt=prompt,  # type: ignore
+            )
+        elif mode == "chat":
+            if messages is None:
+                raise Exception("messages is not set")
+            completion = await client.chat.completions.with_raw_response.create(
+                model=model,  # type: ignore
+                messages=messages,  # type: ignore
+            )
+        elif mode == "embedding":
+            if input is None:
+                raise Exception("input is not set")
+            completion = await client.embeddings.with_raw_response.create(
+                model=model,  # type: ignore
+                input=input,  # type: ignore
+            )
+        elif mode == "image_generation":
+            if prompt is None:
+                raise Exception("prompt is not set")
+            completion = await client.images.with_raw_response.generate(
+                model=model,  # type: ignore
+                prompt=prompt,  # type: ignore
+            )
+        else:
+            raise Exception("mode not set")
+        response = {}
+
+        if completion is None or not hasattr(completion, "headers"):
+            raise Exception("invalid completion response")
+
+        if (
+            completion.headers.get("x-ratelimit-remaining-requests", None) is not None
+        ):  # not provided for dall-e requests
+            response["x-ratelimit-remaining-requests"] = completion.headers[
+                "x-ratelimit-remaining-requests"
+            ]
+
+        if completion.headers.get("x-ratelimit-remaining-tokens", None) is not None:
+            response["x-ratelimit-remaining-tokens"] = completion.headers[
+                "x-ratelimit-remaining-tokens"
+            ]
+        return response

litellm/llms/base.py

@@ -0,0 +1,45 @@
+## This is a template base class to be used for adding new LLM providers via API calls
+import litellm
+import httpx
+from typing import Optional
+
+
+class BaseLLM:
+    _client_session: Optional[httpx.Client] = None
+
+    def create_client_session(self):
+        if litellm.client_session:
+            _client_session = litellm.client_session
+        else:
+            _client_session = httpx.Client()
+
+        return _client_session
+
+    def create_aclient_session(self):
+        if litellm.aclient_session:
+            _aclient_session = litellm.aclient_session
+        else:
+            _aclient_session = httpx.AsyncClient()
+
+        return _aclient_session
+
+    def __exit__(self):
+        if hasattr(self, "_client_session"):
+            self._client_session.close()
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if hasattr(self, "_aclient_session"):
+            await self._aclient_session.aclose()
+
+    def validate_environment(self):  # set up the environment required to run the model
+        pass
+
+    def completion(
+        self, *args, **kwargs
+    ):  # logic for parsing in - calling - parsing out model completion calls
+        pass
+
+    def embedding(
+        self, *args, **kwargs
+    ):  # logic for parsing in - calling - parsing out model embedding calls
+        pass

litellm/llms/baseten.py

@@ -0,0 +1,164 @@
+import os
+import json
+from enum import Enum
+import requests
+import time
+from typing import Callable
+from litellm.utils import ModelResponse, Usage
+
+
+class BasetenError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+def validate_environment(api_key):
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+    }
+    if api_key:
+        headers["Authorization"] = f"Api-Key {api_key}"
+    return headers
+
+
+def completion(
+    model: str,
+    messages: list,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    api_key,
+    logging_obj,
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+):
+    headers = validate_environment(api_key)
+    completion_url_fragment_1 = "https://app.baseten.co/models/"
+    completion_url_fragment_2 = "/predict"
+    model = model
+    prompt = ""
+    for message in messages:
+        if "role" in message:
+            if message["role"] == "user":
+                prompt += f"{message['content']}"
+            else:
+                prompt += f"{message['content']}"
+        else:
+            prompt += f"{message['content']}"
+    data = {
+        "inputs": prompt,
+        "prompt": prompt,
+        "parameters": optional_params,
+        "stream": True
+        if "stream" in optional_params and optional_params["stream"] == True
+        else False,
+    }
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=prompt,
+        api_key=api_key,
+        additional_args={"complete_input_dict": data},
+    )
+    ## COMPLETION CALL
+    response = requests.post(
+        completion_url_fragment_1 + model + completion_url_fragment_2,
+        headers=headers,
+        data=json.dumps(data),
+        stream=True
+        if "stream" in optional_params and optional_params["stream"] == True
+        else False,
+    )
+    if "text/event-stream" in response.headers["Content-Type"] or (
+        "stream" in optional_params and optional_params["stream"] == True
+    ):
+        return response.iter_lines()
+    else:
+        ## LOGGING
+        logging_obj.post_call(
+            input=prompt,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response.text}")
+        ## RESPONSE OBJECT
+        completion_response = response.json()
+        if "error" in completion_response:
+            raise BasetenError(
+                message=completion_response["error"],
+                status_code=response.status_code,
+            )
+        else:
+            if "model_output" in completion_response:
+                if (
+                    isinstance(completion_response["model_output"], dict)
+                    and "data" in completion_response["model_output"]
+                    and isinstance(completion_response["model_output"]["data"], list)
+                ):
+                    model_response["choices"][0]["message"][
+                        "content"
+                    ] = completion_response["model_output"]["data"][0]
+                elif isinstance(completion_response["model_output"], str):
+                    model_response["choices"][0]["message"][
+                        "content"
+                    ] = completion_response["model_output"]
+            elif "completion" in completion_response and isinstance(
+                completion_response["completion"], str
+            ):
+                model_response["choices"][0]["message"][
+                    "content"
+                ] = completion_response["completion"]
+            elif isinstance(completion_response, list) and len(completion_response) > 0:
+                if "generated_text" not in completion_response:
+                    raise BasetenError(
+                        message=f"Unable to parse response. Original response: {response.text}",
+                        status_code=response.status_code,
+                    )
+                model_response["choices"][0]["message"][
+                    "content"
+                ] = completion_response[0]["generated_text"]
+                ## GETTING LOGPROBS
+                if (
+                    "details" in completion_response[0]
+                    and "tokens" in completion_response[0]["details"]
+                ):
+                    model_response.choices[0].finish_reason = completion_response[0][
+                        "details"
+                    ]["finish_reason"]
+                    sum_logprob = 0
+                    for token in completion_response[0]["details"]["tokens"]:
+                        sum_logprob += token["logprob"]
+                    model_response["choices"][0]["message"]._logprobs = sum_logprob
+            else:
+                raise BasetenError(
+                    message=f"Unable to parse response. Original response: {response.text}",
+                    status_code=response.status_code,
+                )
+
+        ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
+        prompt_tokens = len(encoding.encode(prompt))
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"]["content"])
+        )
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        model_response.usage = usage
+        return model_response
+
+
+def embedding():
+    # logic for parsing in - calling - parsing out model embedding calls
+    pass

litellm/llms/bedrock.py

@@ -0,0 +1,799 @@
+import json, copy, types
+import os
+from enum import Enum
+import time
+from typing import Callable, Optional, Any, Union
+import litellm
+from litellm.utils import ModelResponse, get_secret, Usage
+from .prompt_templates.factory import prompt_factory, custom_prompt
+import httpx
+
+
+class BedrockError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(
+            method="POST", url="https://us-west-2.console.aws.amazon.com/bedrock"
+        )
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class AmazonTitanConfig:
+    """
+    Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-text-express-v1
+
+    Supported Params for the Amazon Titan models:
+
+    - `maxTokenCount` (integer) max tokens,
+    - `stopSequences` (string[]) list of stop sequence strings
+    - `temperature` (float) temperature for model,
+    - `topP` (int) top p for model
+    """
+
+    maxTokenCount: Optional[int] = None
+    stopSequences: Optional[list] = None
+    temperature: Optional[float] = None
+    topP: Optional[int] = None
+
+    def __init__(
+        self,
+        maxTokenCount: Optional[int] = None,
+        stopSequences: Optional[list] = None,
+        temperature: Optional[float] = None,
+        topP: Optional[int] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+class AmazonAnthropicConfig:
+    """
+    Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude
+
+    Supported Params for the Amazon / Anthropic models:
+
+    - `max_tokens_to_sample` (integer) max tokens,
+    - `temperature` (float) model temperature,
+    - `top_k` (integer) top k,
+    - `top_p` (integer) top p,
+    - `stop_sequences` (string[]) list of stop sequences - e.g. ["\\n\\nHuman:"],
+    - `anthropic_version` (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31"
+    """
+
+    max_tokens_to_sample: Optional[int] = litellm.max_tokens
+    stop_sequences: Optional[list] = None
+    temperature: Optional[float] = None
+    top_k: Optional[int] = None
+    top_p: Optional[int] = None
+    anthropic_version: Optional[str] = None
+
+    def __init__(
+        self,
+        max_tokens_to_sample: Optional[int] = None,
+        stop_sequences: Optional[list] = None,
+        temperature: Optional[float] = None,
+        top_k: Optional[int] = None,
+        top_p: Optional[int] = None,
+        anthropic_version: Optional[str] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+class AmazonCohereConfig:
+    """
+    Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command
+
+    Supported Params for the Amazon / Cohere models:
+
+    - `max_tokens` (integer) max tokens,
+    - `temperature` (float) model temperature,
+    - `return_likelihood` (string) n/a
+    """
+
+    max_tokens: Optional[int] = None
+    temperature: Optional[float] = None
+    return_likelihood: Optional[str] = None
+
+    def __init__(
+        self,
+        max_tokens: Optional[int] = None,
+        temperature: Optional[float] = None,
+        return_likelihood: Optional[str] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+class AmazonAI21Config:
+    """
+    Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra
+
+    Supported Params for the Amazon / AI21 models:
+
+    - `maxTokens` (int32): The maximum number of tokens to generate per result. Optional, default is 16. If no `stopSequences` are given, generation stops after producing `maxTokens`.
+
+    - `temperature` (float): Modifies the distribution from which tokens are sampled. Optional, default is 0.7. A value of 0 essentially disables sampling and results in greedy decoding.
+
+    - `topP` (float): Used for sampling tokens from the corresponding top percentile of probability mass. Optional, default is 1. For instance, a value of 0.9 considers only tokens comprising the top 90% probability mass.
+
+    - `stopSequences` (array of strings): Stops decoding if any of the input strings is generated. Optional.
+
+    - `frequencyPenalty` (object): Placeholder for frequency penalty object.
+
+    - `presencePenalty` (object): Placeholder for presence penalty object.
+
+    - `countPenalty` (object): Placeholder for count penalty object.
+    """
+
+    maxTokens: Optional[int] = None
+    temperature: Optional[float] = None
+    topP: Optional[float] = None
+    stopSequences: Optional[list] = None
+    frequencePenalty: Optional[dict] = None
+    presencePenalty: Optional[dict] = None
+    countPenalty: Optional[dict] = None
+
+    def __init__(
+        self,
+        maxTokens: Optional[int] = None,
+        temperature: Optional[float] = None,
+        topP: Optional[float] = None,
+        stopSequences: Optional[list] = None,
+        frequencePenalty: Optional[dict] = None,
+        presencePenalty: Optional[dict] = None,
+        countPenalty: Optional[dict] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+class AnthropicConstants(Enum):
+    HUMAN_PROMPT = "\n\nHuman: "
+    AI_PROMPT = "\n\nAssistant: "
+
+
+class AmazonLlamaConfig:
+    """
+    Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=meta.llama2-13b-chat-v1
+
+    Supported Params for the Amazon / Meta Llama models:
+
+    - `max_gen_len` (integer) max tokens,
+    - `temperature` (float) temperature for model,
+    - `top_p` (float) top p for model
+    """
+
+    max_gen_len: Optional[int] = None
+    temperature: Optional[float] = None
+    topP: Optional[float] = None
+
+    def __init__(
+        self,
+        maxTokenCount: Optional[int] = None,
+        temperature: Optional[float] = None,
+        topP: Optional[int] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+def init_bedrock_client(
+    region_name=None,
+    aws_access_key_id: Optional[str] = None,
+    aws_secret_access_key: Optional[str] = None,
+    aws_region_name: Optional[str] = None,
+    aws_bedrock_runtime_endpoint: Optional[str] = None,
+):
+    # check for custom AWS_REGION_NAME and use it if not passed to init_bedrock_client
+    litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
+    standard_aws_region_name = get_secret("AWS_REGION", None)
+
+    ## CHECK IS  'os.environ/' passed in
+    # Define the list of parameters to check
+    params_to_check = [
+        aws_access_key_id,
+        aws_secret_access_key,
+        aws_region_name,
+        aws_bedrock_runtime_endpoint,
+    ]
+
+    # Iterate over parameters and update if needed
+    for i, param in enumerate(params_to_check):
+        if param and param.startswith("os.environ/"):
+            params_to_check[i] = get_secret(param)
+    # Assign updated values back to parameters
+    (
+        aws_access_key_id,
+        aws_secret_access_key,
+        aws_region_name,
+        aws_bedrock_runtime_endpoint,
+    ) = params_to_check
+    if region_name:
+        pass
+    elif aws_region_name:
+        region_name = aws_region_name
+    elif litellm_aws_region_name:
+        region_name = litellm_aws_region_name
+    elif standard_aws_region_name:
+        region_name = standard_aws_region_name
+    else:
+        raise BedrockError(
+            message="AWS region not set: set AWS_REGION_NAME or AWS_REGION env variable or in .env file",
+            status_code=401,
+        )
+
+    # check for custom AWS_BEDROCK_RUNTIME_ENDPOINT and use it if not passed to init_bedrock_client
+    env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT")
+    if aws_bedrock_runtime_endpoint:
+        endpoint_url = aws_bedrock_runtime_endpoint
+    elif env_aws_bedrock_runtime_endpoint:
+        endpoint_url = env_aws_bedrock_runtime_endpoint
+    else:
+        endpoint_url = f"https://bedrock-runtime.{region_name}.amazonaws.com"
+
+    import boto3
+
+    if aws_access_key_id != None:
+        # uses auth params passed to completion
+        # aws_access_key_id is not None, assume user is trying to auth using litellm.completion
+
+        client = boto3.client(
+            service_name="bedrock-runtime",
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            region_name=region_name,
+            endpoint_url=endpoint_url,
+        )
+    else:
+        # aws_access_key_id is None, assume user is trying to auth using env variables
+        # boto3 automatically reads env variables
+
+        client = boto3.client(
+            service_name="bedrock-runtime",
+            region_name=region_name,
+            endpoint_url=endpoint_url,
+        )
+
+    return client
+
+
+def convert_messages_to_prompt(model, messages, provider, custom_prompt_dict):
+    # handle anthropic prompts using anthropic constants
+    if provider == "anthropic":
+        if model in custom_prompt_dict:
+            # check if the model has a registered custom prompt
+            model_prompt_details = custom_prompt_dict[model]
+            prompt = custom_prompt(
+                role_dict=model_prompt_details["roles"],
+                initial_prompt_value=model_prompt_details["initial_prompt_value"],
+                final_prompt_value=model_prompt_details["final_prompt_value"],
+                messages=messages,
+            )
+        else:
+            prompt = prompt_factory(
+                model=model, messages=messages, custom_llm_provider="anthropic"
+            )
+    else:
+        prompt = ""
+        for message in messages:
+            if "role" in message:
+                if message["role"] == "user":
+                    prompt += f"{message['content']}"
+                else:
+                    prompt += f"{message['content']}"
+            else:
+                prompt += f"{message['content']}"
+    return prompt
+
+
+"""
+BEDROCK AUTH Keys/Vars
+os.environ['AWS_ACCESS_KEY_ID'] = ""
+os.environ['AWS_SECRET_ACCESS_KEY'] = ""
+"""
+
+
+# set os.environ['AWS_REGION_NAME'] = <your-region_name>
+
+
+def completion(
+    model: str,
+    messages: list,
+    custom_prompt_dict: dict,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    logging_obj,
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+):
+    exception_mapping_worked = False
+    try:
+        # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
+        aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
+        aws_access_key_id = optional_params.pop("aws_access_key_id", None)
+        aws_region_name = optional_params.pop("aws_region_name", None)
+        aws_bedrock_runtime_endpoint = optional_params.pop(
+            "aws_bedrock_runtime_endpoint", None
+        )
+
+        # use passed in BedrockRuntime.Client if provided, otherwise create a new one
+        client = optional_params.pop("aws_bedrock_client", None)
+
+        # only init client, if user did not pass one
+        if client is None:
+            client = init_bedrock_client(
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                aws_region_name=aws_region_name,
+                aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+            )
+
+        model = model
+        modelId = (
+            optional_params.pop("model_id", None) or model
+        )  # default to model if not passed
+        provider = model.split(".")[0]
+        prompt = convert_messages_to_prompt(
+            model, messages, provider, custom_prompt_dict
+        )
+        inference_params = copy.deepcopy(optional_params)
+        stream = inference_params.pop("stream", False)
+        if provider == "anthropic":
+            ## LOAD CONFIG
+            config = litellm.AmazonAnthropicConfig.get_config()
+            for k, v in config.items():
+                if (
+                    k not in inference_params
+                ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
+                    inference_params[k] = v
+            data = json.dumps({"prompt": prompt, **inference_params})
+        elif provider == "ai21":
+            ## LOAD CONFIG
+            config = litellm.AmazonAI21Config.get_config()
+            for k, v in config.items():
+                if (
+                    k not in inference_params
+                ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
+                    inference_params[k] = v
+
+            data = json.dumps({"prompt": prompt, **inference_params})
+        elif provider == "cohere":
+            ## LOAD CONFIG
+            config = litellm.AmazonCohereConfig.get_config()
+            for k, v in config.items():
+                if (
+                    k not in inference_params
+                ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
+                    inference_params[k] = v
+            if optional_params.get("stream", False) == True:
+                inference_params[
+                    "stream"
+                ] = True  # cohere requires stream = True in inference params
+            data = json.dumps({"prompt": prompt, **inference_params})
+        elif provider == "meta":
+            ## LOAD CONFIG
+            config = litellm.AmazonLlamaConfig.get_config()
+            for k, v in config.items():
+                if (
+                    k not in inference_params
+                ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
+                    inference_params[k] = v
+            data = json.dumps({"prompt": prompt, **inference_params})
+        elif provider == "amazon":  # amazon titan
+            ## LOAD CONFIG
+            config = litellm.AmazonTitanConfig.get_config()
+            for k, v in config.items():
+                if (
+                    k not in inference_params
+                ):  # completion(top_k=3) > amazon_config(top_k=3) <- allows for dynamic variables to be passed in
+                    inference_params[k] = v
+
+            data = json.dumps(
+                {
+                    "inputText": prompt,
+                    "textGenerationConfig": inference_params,
+                }
+            )
+        else:
+            data = json.dumps({})
+
+        ## COMPLETION CALL
+        accept = "application/json"
+        contentType = "application/json"
+        if stream == True:
+            if provider == "ai21":
+                ## LOGGING
+                request_str = f"""
+                response = client.invoke_model(
+                    body={data},
+                    modelId={modelId},
+                    accept=accept,
+                    contentType=contentType
+                )
+                """
+                logging_obj.pre_call(
+                    input=prompt,
+                    api_key="",
+                    additional_args={
+                        "complete_input_dict": data,
+                        "request_str": request_str,
+                    },
+                )
+
+                response = client.invoke_model(
+                    body=data, modelId=modelId, accept=accept, contentType=contentType
+                )
+
+                response = response.get("body").read()
+                return response
+            else:
+                ## LOGGING
+                request_str = f"""
+                response = client.invoke_model_with_response_stream(
+                    body={data},
+                    modelId={modelId},
+                    accept=accept,
+                    contentType=contentType
+                )
+                """
+                logging_obj.pre_call(
+                    input=prompt,
+                    api_key="",
+                    additional_args={
+                        "complete_input_dict": data,
+                        "request_str": request_str,
+                    },
+                )
+
+                response = client.invoke_model_with_response_stream(
+                    body=data, modelId=modelId, accept=accept, contentType=contentType
+                )
+                response = response.get("body")
+                return response
+        try:
+            ## LOGGING
+            request_str = f"""
+            response = client.invoke_model(
+                body={data},
+                modelId={modelId},
+                accept=accept,
+                contentType=contentType
+            )
+            """
+            logging_obj.pre_call(
+                input=prompt,
+                api_key="",
+                additional_args={
+                    "complete_input_dict": data,
+                    "request_str": request_str,
+                },
+            )
+            response = client.invoke_model(
+                body=data, modelId=modelId, accept=accept, contentType=contentType
+            )
+        except client.exceptions.ValidationException as e:
+            if "The provided model identifier is invalid" in str(e):
+                raise BedrockError(status_code=404, message=str(e))
+            raise BedrockError(status_code=400, message=str(e))
+        except Exception as e:
+            raise BedrockError(status_code=500, message=str(e))
+
+        response_body = json.loads(response.get("body").read())
+
+        ## LOGGING
+        logging_obj.post_call(
+            input=prompt,
+            api_key="",
+            original_response=json.dumps(response_body),
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response}")
+        ## RESPONSE OBJECT
+        outputText = "default"
+        if provider == "ai21":
+            outputText = response_body.get("completions")[0].get("data").get("text")
+        elif provider == "anthropic":
+            outputText = response_body["completion"]
+            model_response["finish_reason"] = response_body["stop_reason"]
+        elif provider == "cohere":
+            outputText = response_body["generations"][0]["text"]
+        elif provider == "meta":
+            outputText = response_body["generation"]
+        else:  # amazon titan
+            outputText = response_body.get("results")[0].get("outputText")
+
+        response_metadata = response.get("ResponseMetadata", {})
+        if response_metadata.get("HTTPStatusCode", 500) >= 400:
+            raise BedrockError(
+                message=outputText,
+                status_code=response_metadata.get("HTTPStatusCode", 500),
+            )
+        else:
+            try:
+                if len(outputText) > 0:
+                    model_response["choices"][0]["message"]["content"] = outputText
+            except:
+                raise BedrockError(
+                    message=json.dumps(outputText),
+                    status_code=response_metadata.get("HTTPStatusCode", 500),
+                )
+
+        ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
+        prompt_tokens = len(encoding.encode(prompt))
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+        )
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        model_response.usage = usage
+        return model_response
+    except BedrockError as e:
+        exception_mapping_worked = True
+        raise e
+    except Exception as e:
+        if exception_mapping_worked:
+            raise e
+        else:
+            import traceback
+
+            raise BedrockError(status_code=500, message=traceback.format_exc())
+
+
+def _embedding_func_single(
+    model: str,
+    input: str,
+    client: Any,
+    optional_params=None,
+    encoding=None,
+    logging_obj=None,
+):
+    # logic for parsing in - calling - parsing out model embedding calls
+    ## FORMAT EMBEDDING INPUT ##
+    provider = model.split(".")[0]
+    inference_params = copy.deepcopy(optional_params)
+    inference_params.pop(
+        "user", None
+    )  # make sure user is not passed in for bedrock call
+    modelId = (
+        optional_params.pop("model_id", None) or model
+    )  # default to model if not passed
+    if provider == "amazon":
+        input = input.replace(os.linesep, " ")
+        data = {"inputText": input, **inference_params}
+        # data = json.dumps(data)
+    elif provider == "cohere":
+        inference_params["input_type"] = inference_params.get(
+            "input_type", "search_document"
+        )  # aws bedrock example default - https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/providers?model=cohere.embed-english-v3
+        data = {"texts": [input], **inference_params}  # type: ignore
+    body = json.dumps(data).encode("utf-8")
+    ## LOGGING
+    request_str = f"""
+    response = client.invoke_model(
+        body={body},
+        modelId={modelId},
+        accept="*/*",
+        contentType="application/json",
+    )"""  # type: ignore
+    logging_obj.pre_call(
+        input=input,
+        api_key="",  # boto3 is used for init.
+        additional_args={
+            "complete_input_dict": {"model": modelId, "texts": input},
+            "request_str": request_str,
+        },
+    )
+    try:
+        response = client.invoke_model(
+            body=body,
+            modelId=modelId,
+            accept="*/*",
+            contentType="application/json",
+        )
+        response_body = json.loads(response.get("body").read())
+        ## LOGGING
+        logging_obj.post_call(
+            input=input,
+            api_key="",
+            additional_args={"complete_input_dict": data},
+            original_response=json.dumps(response_body),
+        )
+        if provider == "cohere":
+            response = response_body.get("embeddings")
+            # flatten list
+            response = [item for sublist in response for item in sublist]
+            return response
+        elif provider == "amazon":
+            return response_body.get("embedding")
+    except Exception as e:
+        raise BedrockError(
+            message=f"Embedding Error with model {model}: {e}", status_code=500
+        )
+
+
+def embedding(
+    model: str,
+    input: Union[list, str],
+    api_key: Optional[str] = None,
+    logging_obj=None,
+    model_response=None,
+    optional_params=None,
+    encoding=None,
+):
+    ### BOTO3 INIT ###
+    # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
+    aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
+    aws_access_key_id = optional_params.pop("aws_access_key_id", None)
+    aws_region_name = optional_params.pop("aws_region_name", None)
+    aws_bedrock_runtime_endpoint = optional_params.pop(
+        "aws_bedrock_runtime_endpoint", None
+    )
+
+    # use passed in BedrockRuntime.Client if provided, otherwise create a new one
+    client = init_bedrock_client(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        aws_region_name=aws_region_name,
+        aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+    )
+    if type(input) == str:
+        embeddings = [
+            _embedding_func_single(
+                model,
+                input,
+                optional_params=optional_params,
+                client=client,
+                logging_obj=logging_obj,
+            )
+        ]
+    else:
+        ## Embedding Call
+        embeddings = [
+            _embedding_func_single(
+                model,
+                i,
+                optional_params=optional_params,
+                client=client,
+                logging_obj=logging_obj,
+            )
+            for i in input
+        ]  # [TODO]: make these parallel calls
+
+    ## Populate OpenAI compliant dictionary
+    embedding_response = []
+    for idx, embedding in enumerate(embeddings):
+        embedding_response.append(
+            {
+                "object": "embedding",
+                "index": idx,
+                "embedding": embedding,
+            }
+        )
+    model_response["object"] = "list"
+    model_response["data"] = embedding_response
+    model_response["model"] = model
+    input_tokens = 0
+
+    input_str = "".join(input)
+
+    input_tokens += len(encoding.encode(input_str))
+
+    usage = Usage(
+        prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens + 0
+    )
+    model_response.usage = usage
+
+    return model_response

litellm/llms/cloudflare.py

@@ -0,0 +1,176 @@
+import os, types
+import json
+from enum import Enum
+import requests
+import time
+from typing import Callable, Optional
+import litellm
+import httpx
+from litellm.utils import ModelResponse, Usage
+from .prompt_templates.factory import prompt_factory, custom_prompt
+
+
+class CloudflareError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(method="POST", url="https://api.cloudflare.com")
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class CloudflareConfig:
+    max_tokens: Optional[int] = None
+    stream: Optional[bool] = None
+
+    def __init__(
+        self,
+        max_tokens: Optional[int] = None,
+        stream: Optional[bool] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+def validate_environment(api_key):
+    if api_key is None:
+        raise ValueError(
+            "Missing CloudflareError API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params"
+        )
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "Authorization": "Bearer " + api_key,
+    }
+    return headers
+
+
+def completion(
+    model: str,
+    messages: list,
+    api_base: str,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    api_key,
+    logging_obj,
+    custom_prompt_dict={},
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+):
+    headers = validate_environment(api_key)
+
+    ## Load Config
+    config = litellm.CloudflareConfig.get_config()
+    for k, v in config.items():
+        if k not in optional_params:
+            optional_params[k] = v
+
+    print_verbose(f"CUSTOM PROMPT DICT: {custom_prompt_dict}; model: {model}")
+    if model in custom_prompt_dict:
+        # check if the model has a registered custom prompt
+        model_prompt_details = custom_prompt_dict[model]
+        prompt = custom_prompt(
+            role_dict=model_prompt_details.get("roles", {}),
+            initial_prompt_value=model_prompt_details.get("initial_prompt_value", ""),
+            final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
+            bos_token=model_prompt_details.get("bos_token", ""),
+            eos_token=model_prompt_details.get("eos_token", ""),
+            messages=messages,
+        )
+
+    # cloudflare adds the model to the api base
+    api_base = api_base + model
+
+    data = {
+        "messages": messages,
+        **optional_params,
+    }
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=messages,
+        api_key=api_key,
+        additional_args={
+            "headers": headers,
+            "api_base": api_base,
+            "complete_input_dict": data,
+        },
+    )
+
+    ## COMPLETION CALL
+    if "stream" in optional_params and optional_params["stream"] == True:
+        response = requests.post(
+            api_base,
+            headers=headers,
+            data=json.dumps(data),
+            stream=optional_params["stream"],
+        )
+        return response.iter_lines()
+    else:
+        response = requests.post(api_base, headers=headers, data=json.dumps(data))
+        ## LOGGING
+        logging_obj.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response.text}")
+        ## RESPONSE OBJECT
+        if response.status_code != 200:
+            raise CloudflareError(
+                status_code=response.status_code, message=response.text
+            )
+        completion_response = response.json()
+
+        model_response["choices"][0]["message"]["content"] = completion_response[
+            "result"
+        ]["response"]
+
+        ## CALCULATING USAGE
+        print_verbose(
+            f"CALCULATING CLOUDFLARE TOKEN USAGE. Model Response: {model_response}; model_response['choices'][0]['message'].get('content', ''): {model_response['choices'][0]['message'].get('content', None)}"
+        )
+        prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model)
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+        )
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = "cloudflare/" + model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        model_response.usage = usage
+        return model_response
+
+
+def embedding():
+    # logic for parsing in - calling - parsing out model embedding calls
+    pass

litellm/llms/cohere.py

@@ -0,0 +1,293 @@
+import os, types
+import json
+from enum import Enum
+import requests
+import time, traceback
+from typing import Callable, Optional
+from litellm.utils import ModelResponse, Choices, Message, Usage
+import litellm
+import httpx
+
+
+class CohereError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(
+            method="POST", url="https://api.cohere.ai/v1/generate"
+        )
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            self.message
+        )  # Call the base class constructor with the parameters it needs
+
+
+class CohereConfig:
+    """
+    Reference: https://docs.cohere.com/reference/generate
+
+    The class `CohereConfig` provides configuration for the Cohere's API interface. Below are the parameters:
+
+    - `num_generations` (integer): Maximum number of generations returned. Default is 1, with a minimum value of 1 and a maximum value of 5.
+
+    - `max_tokens` (integer): Maximum number of tokens the model will generate as part of the response. Default value is 20.
+
+    - `truncate` (string): Specifies how the API handles inputs longer than maximum token length. Options include NONE, START, END. Default is END.
+
+    - `temperature` (number): A non-negative float controlling the randomness in generation. Lower temperatures result in less random generations. Default is 0.75.
+
+    - `preset` (string): Identifier of a custom preset, a combination of parameters such as prompt, temperature etc.
+
+    - `end_sequences` (array of strings): The generated text gets cut at the beginning of the earliest occurrence of an end sequence, which will be excluded from the text.
+
+    - `stop_sequences` (array of strings): The generated text gets cut at the end of the earliest occurrence of a stop sequence, which will be included in the text.
+
+    - `k` (integer): Limits generation at each step to top `k` most likely tokens. Default is 0.
+
+    - `p` (number): Limits generation at each step to most likely tokens with total probability mass of `p`. Default is 0.
+
+    - `frequency_penalty` (number): Reduces repetitiveness of generated tokens. Higher values apply stronger penalties to previously occurred tokens.
+
+    - `presence_penalty` (number): Reduces repetitiveness of generated tokens. Similar to frequency_penalty, but this penalty applies equally to all tokens that have already appeared.
+
+    - `return_likelihoods` (string): Specifies how and if token likelihoods are returned with the response. Options include GENERATION, ALL and NONE.
+
+    - `logit_bias` (object): Used to prevent the model from generating unwanted tokens or to incentivize it to include desired tokens. e.g. {"hello_world": 1233}
+    """
+
+    num_generations: Optional[int] = None
+    max_tokens: Optional[int] = None
+    truncate: Optional[str] = None
+    temperature: Optional[int] = None
+    preset: Optional[str] = None
+    end_sequences: Optional[list] = None
+    stop_sequences: Optional[list] = None
+    k: Optional[int] = None
+    p: Optional[int] = None
+    frequency_penalty: Optional[int] = None
+    presence_penalty: Optional[int] = None
+    return_likelihoods: Optional[str] = None
+    logit_bias: Optional[dict] = None
+
+    def __init__(
+        self,
+        num_generations: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        truncate: Optional[str] = None,
+        temperature: Optional[int] = None,
+        preset: Optional[str] = None,
+        end_sequences: Optional[list] = None,
+        stop_sequences: Optional[list] = None,
+        k: Optional[int] = None,
+        p: Optional[int] = None,
+        frequency_penalty: Optional[int] = None,
+        presence_penalty: Optional[int] = None,
+        return_likelihoods: Optional[str] = None,
+        logit_bias: Optional[dict] = None,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+
+def validate_environment(api_key):
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+    }
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    return headers
+
+
+def completion(
+    model: str,
+    messages: list,
+    api_base: str,
+    model_response: ModelResponse,
+    print_verbose: Callable,
+    encoding,
+    api_key,
+    logging_obj,
+    optional_params=None,
+    litellm_params=None,
+    logger_fn=None,
+):
+    headers = validate_environment(api_key)
+    completion_url = api_base
+    model = model
+    prompt = " ".join(message["content"] for message in messages)
+
+    ## Load Config
+    config = litellm.CohereConfig.get_config()
+    for k, v in config.items():
+        if (
+            k not in optional_params
+        ):  # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in
+            optional_params[k] = v
+
+    data = {
+        "model": model,
+        "prompt": prompt,
+        **optional_params,
+    }
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=prompt,
+        api_key=api_key,
+        additional_args={
+            "complete_input_dict": data,
+            "headers": headers,
+            "api_base": completion_url,
+        },
+    )
+    ## COMPLETION CALL
+    response = requests.post(
+        completion_url,
+        headers=headers,
+        data=json.dumps(data),
+        stream=optional_params["stream"] if "stream" in optional_params else False,
+    )
+    ## error handling for cohere calls
+    if response.status_code != 200:
+        raise CohereError(message=response.text, status_code=response.status_code)
+
+    if "stream" in optional_params and optional_params["stream"] == True:
+        return response.iter_lines()
+    else:
+        ## LOGGING
+        logging_obj.post_call(
+            input=prompt,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response.text}")
+        ## RESPONSE OBJECT
+        completion_response = response.json()
+        if "error" in completion_response:
+            raise CohereError(
+                message=completion_response["error"],
+                status_code=response.status_code,
+            )
+        else:
+            try:
+                choices_list = []
+                for idx, item in enumerate(completion_response["generations"]):
+                    if len(item["text"]) > 0:
+                        message_obj = Message(content=item["text"])
+                    else:
+                        message_obj = Message(content=None)
+                    choice_obj = Choices(
+                        finish_reason=item["finish_reason"],
+                        index=idx + 1,
+                        message=message_obj,
+                    )
+                    choices_list.append(choice_obj)
+                model_response["choices"] = choices_list
+            except Exception as e:
+                raise CohereError(
+                    message=response.text, status_code=response.status_code
+                )
+
+        ## CALCULATING USAGE
+        prompt_tokens = len(encoding.encode(prompt))
+        completion_tokens = len(
+            encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+        )
+
+        model_response["created"] = int(time.time())
+        model_response["model"] = model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        model_response.usage = usage
+        return model_response
+
+
+def embedding(
+    model: str,
+    input: list,
+    api_key: Optional[str] = None,
+    logging_obj=None,
+    model_response=None,
+    encoding=None,
+    optional_params=None,
+):
+    headers = validate_environment(api_key)
+    embed_url = "https://api.cohere.ai/v1/embed"
+    model = model
+    data = {"model": model, "texts": input, **optional_params}
+
+    if "3" in model and "input_type" not in data:
+        # cohere v3 embedding models require input_type, if no input_type is provided, default to "search_document"
+        data["input_type"] = "search_document"
+
+    ## LOGGING
+    logging_obj.pre_call(
+        input=input,
+        api_key=api_key,
+        additional_args={"complete_input_dict": data},
+    )
+    ## COMPLETION CALL
+    response = requests.post(embed_url, headers=headers, data=json.dumps(data))
+    ## LOGGING
+    logging_obj.post_call(
+        input=input,
+        api_key=api_key,
+        additional_args={"complete_input_dict": data},
+        original_response=response,
+    )
+    """
+        response 
+        {
+            'object': "list",
+            'data': [
+            
+            ]
+            'model', 
+            'usage'
+        }
+    """
+    if response.status_code != 200:
+        raise CohereError(message=response.text, status_code=response.status_code)
+    embeddings = response.json()["embeddings"]
+    output_data = []
+    for idx, embedding in enumerate(embeddings):
+        output_data.append(
+            {"object": "embedding", "index": idx, "embedding": embedding}
+        )
+    model_response["object"] = "list"
+    model_response["data"] = output_data
+    model_response["model"] = model
+    input_tokens = 0
+    for text in input:
+        input_tokens += len(encoding.encode(text))
+
+    model_response["usage"] = {
+        "prompt_tokens": input_tokens,
+        "total_tokens": input_tokens,
+    }
+    return model_response