Repository: superduper-io/superduper

Issue:
[LLM] Support for Llama.cpp and local deployment with vLLM
We have already built support for vLLM, which will work well in the cloud/ GPU machines/ Linux machines/ data centers.
However, on systems such as Mac OSX, we are missing a solution. 

For this reason, we would like to integrate [Llama.cpp](https://github.com/ggerganov/llama.cpp). Below we discuss various options and paths to doing this.

Investigate potential bugs using transformers & vLLM on Mac.


Discussion:
We can quick implement the new llm model like this 
https://docs.superduperdb.com/docs/docs/ai_integrations/llm#custom-models

Resolving patch:
diff --git a/superduperdb/ext/llamacpp/__init__.py b/superduperdb/ext/llamacpp/__init__.py
new file mode 100644
--- /dev/null
+++ b/superduperdb/ext/llamacpp/__init__.py
@@ -0,0 +1 @@
+
diff --git a/superduperdb/ext/llamacpp/model.py b/superduperdb/ext/llamacpp/model.py
new file mode 100644
--- /dev/null
+++ b/superduperdb/ext/llamacpp/model.py
@@ -0,0 +1,56 @@
+import dataclasses as dc
+import os
+import typing as t
+
+import requests
+from llama_cpp import Llama
+
+from superduperdb.ext.llm.base import _BaseLLM
+
+
+def download_uri(uri, save_path):
+    response = requests.get(uri)
+    if response.status_code == 200:
+        with open(save_path, 'wb') as file:
+            file.write(response.content)
+    else:
+        raise Exception(f"Error while downloading uri {uri}")
+
+
+@dc.dataclass
+class LlamaCpp(_BaseLLM):
+    model_name_or_path: str = "facebook/opt-125m"
+    object: t.Optional[Llama] = None
+    model_kwargs: t.Dict = dc.field(default_factory=dict)
+    download_dir: str = '.llama_cpp'
+
+    def init(self):
+        if self.model_name_or_path.startswith('http'):
+            # Download the uri
+            os.makedirs(self.download_dir, exist_ok=True)
+            saved_path = os.path.join(self.download_dir, f'{self.identifier}.gguf')
+
+            download_uri(self.model_name_or_path, saved_path)
+            self.model_name_or_path = saved_path
+
+        if self.predict_kwargs is None:
+            self.predict_kwargs = {}
+
+        self._model = Llama(self.model_name_or_path, **self.model_kwargs)
+
+    def _generate(self, prompt: str, **kwargs) -> str:
+        """
+        Generate text from a prompt.
+        """
+        return self._model.create_completion(prompt, **self.predict_kwargs, **kwargs)
+
+
+@dc.dataclass
+class LlamaCppEmbedding(LlamaCpp):
+    def _generate(self, prompt: str, **kwargs) -> str:
+        """
+        Generate embedding from a prompt.
+        """
+        return self._model.create_embedding(
+            prompt, embedding=True, **self.predict_kwargs, **kwargs
+        )


Test patch:
diff --git a/test/unittest/ext/test_llama_cpp.py b/test/unittest/ext/test_llama_cpp.py
new file mode 100644
--- /dev/null
+++ b/test/unittest/ext/test_llama_cpp.py
@@ -0,0 +1,45 @@
+from superduperdb.ext.llamacpp.model import LlamaCpp, LlamaCppEmbedding
+
+
+class _MockedLlama:
+    def create_completion(self, *args, **kwargs):
+        return 'tested'
+
+    def create_embedding(self, *args, **kwargs):
+        return [1]
+
+
+def test_llama():
+    def mocked_init(self):
+        self._model = _MockedLlama()
+        self.predict_kwargs = {}
+
+    LlamaCpp.init = mocked_init
+
+    llama = LlamaCpp(
+        identifier='myllama',
+        model_name_or_path='some_model',
+        model_kwargs={'vocab_only': True},
+    )
+
+    text = 'testing prompt'
+    output = llama.predict(text, one=True)
+    assert output == 'tested'
+
+
+def test_llama_embedding():
+    def mocked_init(self):
+        self._model = _MockedLlama()
+        self.predict_kwargs = {}
+
+    LlamaCppEmbedding.init = mocked_init
+
+    llama = LlamaCppEmbedding(
+        identifier='myllama',
+        model_name_or_path='some_model',
+        model_kwargs={'vocab_only': True},
+    )
+
+    text = 'testing prompt'
+    output = llama.predict(text, one=True)
+    assert output == [1]