litellm_config.yaml

model_list:
  - model_name: hf-mistral-7b-instruct-v03
    litellm_params:
      model: huggingface/mistralai/Mistral-7B-Instruct-v0.3
      api_key: os.environ/HUGGINGFACE_API_KEY
      #api_base: "https://api-inference.huggingface.co/models/"
      #api_base: "https://my-endpoint.endpoints.huggingface.cloud"
  - model_name: awan-llama3-8b
    litellm_params:
      model: openai/Meta-Llama-3-8B-Instruct
      api_key: os.environ/OTHER_AWANLLM_API_KEY
      api_base: https://api.awanllm.com/v1
      rpm: 10
  - model_name: groq-llama3-70b
    litellm_params:
      model: openai/llama3-70b-8192
      api_key: os.environ/GROQ_API_KEY
      api_base: https://api.groq.com/openai/v1
      rpm: 30

litellm_settings:
  success_callback: ["langfuse"]
  failure_callback: ["langfuse"]
  num_retries: 3
  request_timeout: 20
  allowed_fails: 3