Repository: halluxdev/hallux (language: python)

Issue:
Make use of LightLLM
https://github.com/BerriAI/litellm

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
Make use of LightLLM
https://github.com/BerriAI/litellm

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
Make use of LightLLM
https://github.com/BerriAI/litellm

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
Make use of LightLLM
https://github.com/BerriAI/litellm

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
Make use of LightLLM
https://github.com/BerriAI/litellm

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]


Discussion / hints:

Fix patch:
diff --git a/.hallux b/.hallux
index 0c415d1..2f38b74 100644
--- a/.hallux
+++ b/.hallux
@@ -14,11 +14,9 @@ backends:
     #        API-KEY: <KEY>
 
 
-
     - gpt3:
-        type: openai
+        type: litellm
         model: gpt-3.5-turbo
-        max_tokens: 4096
 
 tools:
     #ruff:
diff --git a/README.md b/README.md
index 536aa0b..dcb0b30 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ hallux
 
 In order to setup Python virtual environment run `./scripts/setup-venv.sh`
 
-For activating environment run `cd /path/to/hallux && source activate.sh` 
+For activating environment run `source ./activate.sh` 
 
 
 ## Folder structure
diff --git a/hallux/backends/factory.py b/hallux/backends/factory.py
index 77b29b8..57f4fae 100644
--- a/hallux/backends/factory.py
+++ b/hallux/backends/factory.py
@@ -1,21 +1,21 @@
 # Copyright: Hallux team, 2023
 
 from __future__ import annotations
-
 from pathlib import Path
+from hallux.logger import logger
 
 from ..auxilary import find_arg
 from ..backends.dummy_backend import DummyBackend
-from ..backends.openai_backend import OpenAiChatGPT, QueryBackend
+from ..backends.query_backend import QueryBackend
 from ..backends.rest_backend import RestBackend
-
+from ..backends.litellm import LiteLLMBackend
 
 class BackendFactory:
     @staticmethod
     def init_backend(argv: list[str], backends_list: list[dict] | None, config_path: Path) -> QueryBackend:
         default_list = [
             {"cache": {"type": "dummy", "filename": "dummy.json"}},
-            {"gpt3": {"type": "openai", "model": "gpt-3.5-turbo", "max_tokens": 4096}},
+            {"gpt3": {"type": "litellm", "model": "gpt-3.5-turbo"}},
         ]
 
         backends_list: list[dict] = backends_list if backends_list is not None else default_list
@@ -38,20 +38,39 @@ def _validate_settings(name_dict: dict) -> tuple[str, dict[str, str]]:
         name = str(list(name_dict)[0])
         settings = name_dict[name]
 
-        if not isinstance(settings, dict) or "type" not in settings.keys():
-            raise SystemError(f"Each BACKEND setting must have at least 'type' setting. Error in: {settings}")
+        if not isinstance(settings, dict) or ("type" not in settings.keys() and not settings["model"]):
+            raise SystemError(f"Each BACKEND setting must have at least 'type' or a 'model' setting. Error in: {settings}")
         return name, settings
 
     @staticmethod
     def _create_backend(settings: dict, config_path: Path, previous_backend: QueryBackend) -> QueryBackend:
         type_to_class = {
             "dummy": DummyBackend,
-            "openai": OpenAiChatGPT,
-            "openai.azure": OpenAiChatGPT,
             "rest": RestBackend,
+            "litellm": LiteLLMBackend,
         }
-        backend_type = settings["type"]
+
+        backend_type = settings.get("type", "").strip()
+
+        # Set litellm as the default type if none is specified
+        if not backend_type:
+            backend_type = "litellm"
+            logger.info("No backend type specified. Using 'litellm' as a default backend type.")
+
+
+        # Add backward compatibility for openai and openai.azure types
+        # TODO: remove this in the future versions
+        if backend_type in ["openai", "openai.azure"]:
+            backend_type = "litellm"
+            settings["type"] = "litellm"
+            logger.warning(f"'openai' and 'openai.azure' backends are deprecated. Please use 'litellm' instead.")
+
+        if backend_type in ["openai.azure"]:
+            settings["model"] = "azure/" + settings["model"]
+            logger.warning(f"Model name for 'openai.azure' updated to 'azure/{settings['model']}'.")
+            
+
         backend_class = type_to_class.get(backend_type)
         if backend_class is None:
-            raise SystemError(f"Unknown BACKEND type: {backend_type}")
+            raise SystemError(f"Unknown BACKEND type: {backend_type}. Supported types: {type_to_class.keys()}")
         return backend_class(**settings, base_path=config_path, previous_backend=previous_backend)
diff --git a/hallux/backends/litellm.py b/hallux/backends/litellm.py
new file mode 100644
index 0000000..75128be
--- /dev/null
+++ b/hallux/backends/litellm.py
@@ -0,0 +1,71 @@
+# Copyright: Hallux team, 2023
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Final
+
+from litellm import completion
+
+from hallux.logger import logger
+
+from .query_backend import QueryBackend
+from ..issues.issue import IssueDescriptor
+
+
+class LiteLLMBackend(QueryBackend):
+    def __init__(
+        self,
+        model: str = "",
+        type="litellm",
+        base_path: Path = Path(),
+        previous_backend: QueryBackend | None = None,
+    ):
+        assert type == "litellm"
+        super().__init__(base_path, previous_backend)
+
+        self.valid = True
+        if model is None or len(model) < 2:
+            logger.warning(f"Wrong model name for LiteLLM Backend: {model}")
+            self.valid = False
+
+        self.model: Final[str] = model
+
+
+    def query(self, request: str, issue: IssueDescriptor | None = None, issue_lines: list[str] = list) -> list[str]:
+        if not self.valid:
+            return []
+
+        logger.debug("")
+        logger.debug("[LiteLLM REQUEST]:")
+        for line in request.split("\n"):
+            logger.debug(line)
+
+
+        result = completion(
+            model=self.model,
+            messages=[{ "content": request, "role": "user"}]
+        )
+
+
+        answers = []
+
+        if len(result["choices"]) > 0:
+            for variant in result["choices"]:
+                answers.append(variant["message"]["content"])
+
+        logger.debug("")
+        logger.debug("[LiteLLM ANSWERS]:")
+        for ans in answers:
+            for line in ans.split("\n"):
+                logger.debug(line)
+
+        return answers
+
+    def validate_env(self, key: str, message: str) -> None | str:
+        if os.getenv(key) is None:
+            logger.warning(message.format(key=key))
+            self.valid = False
+        else:
+            return os.getenv(key)
diff --git a/hallux/backends/openai_backend.py b/hallux/backends/openai_backend.py
index 8f020b2..9ecbec1 100644
--- a/hallux/backends/openai_backend.py
+++ b/hallux/backends/openai_backend.py
@@ -7,7 +7,7 @@
 from typing import Final
 
 import openai
-from openai.api_resources import ChatCompletion
+from openai import ChatCompletion
 
 from hallux.logger import logger
 
@@ -21,6 +21,7 @@ def __init__(
         model: str = "",
         max_tokens: int = 4097,
         type="openai",
+        api_version="2021-05-15",
         base_path: Path = Path(),
         previous_backend: QueryBackend | None = None,
     ):
@@ -34,6 +35,7 @@ def __init__(
 
         self.model: Final[str] = model
         self.max_tokens: Final[int] = max_tokens
+        self.api_version: Final[int] = api_version
 
         # For Azure OpenAI
         if type == "openai.azure":
@@ -56,7 +58,9 @@ def query(self, request: str, issue: IssueDescriptor | None = None, issue_lines:
 
         logger.debug("[OpenAI REQUEST]:")
         logger.debug(request)
-        result = ChatCompletion.create(messages=[{"role": "user", "content": request}], model=self.model)
+        result = ChatCompletion.create(messages=[
+            {"role": "user", "content": request}
+        ], model=self.model, engine=self.model)
         answers = []
         if len(result["choices"]) > 0:
             for variant in result["choices"]:
diff --git a/pyproject.toml b/pyproject.toml
index d4d29d7..d7bab63 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,19 +24,19 @@ classifiers = [
 ]
 
 dependencies = [
-  'openai==0.27.8',
   'pyyaml==6.0',
   'types-PyYAML==6.0.12.11',
   'PyGithub==2.1.1',
   'colorlog==6.7.0',
-  'requests==2.31.0',
-  'unidiff==0.7.5'
+  'requests==2.32.0',
+  'unidiff==0.7.5',
+  'litellm==1.41.2'
 ]
 
 [project.optional-dependencies]
 test = [
   'mypy==1.4.1',
-  'black==23.3.0',
+  'black==24.3.0',
   'ruff==0.0.272',
   'pytest >= 7.3.1',
   'pytest-cov==4.1.0',
diff --git a/requirements.txt b/requirements.txt
index 990eb8f..f624463 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,8 +2,7 @@ pytest==7.3.1
 pytest-cov==4.1.0
 ruff==0.0.272
 mypy==1.4.1
-black==23.3.0
-openai==0.27.8
+black==24.3.0
 pyyaml==6.0
 types-PyYAML==6.0.12.11
 PyGithub==2.1.1
@@ -11,5 +10,6 @@ isort==5.12.0
 wheel==0.40.0
 pyinstaller>=5.13.1
 colorlog==6.7.0
-requests==2.31.0
+requests==2.32.0
 unidiff==0.7.5
+litellm==1.41.2


Test patch:
diff --git a/tests/test-python-project/.hallux b/tests/test-python-project/.hallux
index e681a2e..e2e6a87 100644
--- a/tests/test-python-project/.hallux
+++ b/tests/test-python-project/.hallux
@@ -3,11 +3,5 @@ backends:
         type: dummy
         filename: dummy.json
 
-    - rest:
-        type: rest
-        url: http://localhost:8181/generate
-        request_body: { message: Think in step by step manner. $PROMPT } # Defaults to: $PROMPT
-        response_body: $RESPONSE.answer.0.value # Defaults to $RESPONSE
-
 python:
     ruff: .
diff --git a/tests/test-python-project/.hallux.real b/tests/test-python-project/.hallux.real
index 9d87269..677dd40 100644
--- a/tests/test-python-project/.hallux.real
+++ b/tests/test-python-project/.hallux.real
@@ -1,8 +1,12 @@
 backends:
-    - gpt3:
-        type: openai
-        model: gpt-3.5-turbo
-        tokens: 4097
+    - litellm:
+        model: gpt-4o
+
+    - rest:
+        type: rest
+        url: http://localhost:8181/generate
+        request_body: { message: Think in step by step manner. $PROMPT } Defaults to: $PROMPT
+        response_body: $RESPONSE.answer.0.value # Defaults to $RESPONSE
 
 python:
     ruff: python
diff --git a/tests/unit/backends/litellm_test.py b/tests/unit/backends/litellm_test.py
new file mode 100644
index 0000000..9b6aff7
--- /dev/null
+++ b/tests/unit/backends/litellm_test.py
@@ -0,0 +1,68 @@
+import os
+import pytest
+from pathlib import Path
+from hallux.backends.litellm import LiteLLMBackend
+
+from unittest.mock import patch, MagicMock
+from pathlib import Path
+from hallux.issues.issue import IssueDescriptor
+from unittest.mock import patch
+
+
+@pytest.fixture
+def valid_model():
+    return "gpt-4o"
+
+@pytest.fixture
+def invalid_model():
+    return "x"
+
+@pytest.fixture
+def setup_litellm_backend(valid_model):
+    with patch.dict(os.environ, {"OPENAI_API_KEY": "test_key"}):
+        return LiteLLMBackend(model=valid_model)
+
+def test_initialization_valid_model(setup_litellm_backend, valid_model):
+    assert setup_litellm_backend.valid
+    assert setup_litellm_backend.model == valid_model
+
+def test_initialization_invalid_model(invalid_model):
+    with patch('hallux.logger.logger.warning') as mock_warning:
+        backend = LiteLLMBackend(model=invalid_model)
+        assert not backend.valid
+        mock_warning.assert_called_once_with(f"Wrong model name for LiteLLM Backend: {invalid_model}")
+
+@patch('hallux.backends.litellm.completion')
+def test_query_valid_model(mock_completion, setup_litellm_backend):
+    request = "Test request"
+    expected_response = ["Test response"]
+
+    mock_completion.return_value = {"choices": [{"message": {"content": "Test response"}}]}
+    response = setup_litellm_backend.query(request)
+    assert response == expected_response
+
+def test_query_invalid_model(invalid_model):
+    with patch('hallux.logger.logger.warning'):
+        backend = LiteLLMBackend(model=invalid_model)
+    request = "Test request"
+    response = backend.query(request)
+    assert response == []
+
+def test_validate_env_present(monkeypatch):
+    key = "TEST_ENV"
+    value = "test_value"
+    monkeypatch.setenv(key, value)
+    backend = LiteLLMBackend(model="valid_model")
+    result = backend.validate_env(key, "Warning: {key} is not set")
+    assert result == value
+
+def test_validate_env_missing(monkeypatch):
+    key = "MISSING_ENV"
+    message = "Warning: {key} is not set"
+    monkeypatch.delenv(key, raising=False)
+    
+    with patch('hallux.logger.logger.warning') as mock_warning:
+        backend = LiteLLMBackend(model="valid_model")
+        result = backend.validate_env(key, message)
+        assert result is None
+        mock_warning.assert_called_once_with(message.format(key=key))