Repository: Quansight/ragna
Pull request #348 (2024-03-11 09:47:33)

fix typo in FAQ
uses cases -> use cases


Patch:
diff --git a/README.md b/README.md
index a60c1b3..807701e 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,7 @@ A RAG orchestration framework.
 
 Check out the Ragna documentation to get started:
 
+- [Overview video](https://youtu.be/xuNnjFS1XqQ?si=AssQq_1CF2o1jv3M)
 - [Installation steps](https://ragna.chat/en/stable/install/)
 - Tutorials
   - [Python API](https://ragna.chat/en/stable/tutorials/python-api/)
diff --git a/docs/examples/gallery_streaming.py b/docs/examples/gallery_streaming.py
index 43d56f6..cdba77c 100644
--- a/docs/examples/gallery_streaming.py
+++ b/docs/examples/gallery_streaming.py
@@ -43,10 +43,6 @@ from ragna import assistants
 
 
 class DemoStreamingAssistant(assistants.RagnaDemoAssistant):
-    @property
-    def max_input_size(self) -> int:
-        return 0
-
     def answer(self, prompt, sources):
         content = next(super().answer(prompt, sources))
         for chunk in content.split(" "):
diff --git a/docs/index.md b/docs/index.md
index fbeb289..e2d365b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -2,14 +2,14 @@
 title: Home
 ---
 
-# ![Ragna logo](assets/brand/logo-lockup-horizontal/logo-lockup-horizontal.png)
-
-Ragna is an open source RAG orchestration framework.
+# Ragna — Open source RAG orchestration framework
 
 With an intuitive API for quick experimentation and built-in tools for creating
 production-ready application, you can quickly leverage Large Language Models (LLMs) for
 your work.
 
+<iframe width="560" height="315" src="https://www.youtube.com/embed/xuNnjFS1XqQ?si=zKMVXSSjQWZvYvkf" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+
 !!! note
 
     Ragna is a framework for experimenting with LLMs. It does not install one by default. To use Ragna you either need to provide API keys to a cloud LLM API, e.g. OpenAI, Anthropic, etc. or run your own LLM.
diff --git a/docs/references/faq.md b/docs/references/faq.md
index c89376a..d6ebfee 100644
--- a/docs/references/faq.md
+++ b/docs/references/faq.md
@@ -4,8 +4,9 @@
 
 !!! tip "TL;DR"
 
-    Ragna is the only tool out there that specializes in orchestrating RAG uses cases with arbitrary components,
-    as well as offers a Python API, a REST API, and a web UI for that.
+    Ragna is the only tool out there that specializes in orchestrating RAG use cases
+    with arbitrary components, as well as offering a Python API, a REST API, and a web
+    UI for that.
 
 !!! note
 
diff --git a/docs/references/release-notes.md b/docs/references/release-notes.md
index f62bba0..9f4801d 100644
--- a/docs/references/release-notes.md
+++ b/docs/references/release-notes.md
@@ -39,6 +39,9 @@
 - The return type of [ragna.core.Assistant.answer][] changed from `str` to
   `Iterator[str]` / `AsyncIterator[str]`. To reflect that in the implementation, replace
   `return` with `yield`. To stream the response, `yield` multiple times.
+- The abstract property `ragna.core.Assistant.max_input_size` was removed. This
+  information was never used anywhere. Note that you don't have to remove it from
+  existing implementations. It will just stay unused by Ragna.
 - We introduced a couple of changes to the configuration file.
 
   - The top level `local_cache_root` option was renamed to `local_root` to align with
diff --git a/environment-dev.yml b/environment-dev.yml
index cb4bdd4..ca1daaf 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -9,6 +9,7 @@ dependencies:
       - python-dotenv
       - pytest >=6
       - pytest-mock
+      - pytest-asyncio
       - mypy ==1.6.1
       - pre-commit
       - types-aiofiles
diff --git a/pyproject.toml b/pyproject.toml
index 2a5c5bf..574813c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -97,16 +97,17 @@ ignore = ["E501"]
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "-ra --tb=short"
+addopts = "-ra --tb=short --asyncio-mode=auto"
 testpaths = [
     "tests",
 ]
 filterwarnings = [
     "error",
+    "ignore::ResourceWarning",
     # httpx 0.27.0 deprecated some functionality that the test client of starlette /
     # FastApi use. This should be resolved by the next release of these libraries.
     # See https://github.com/encode/starlette/issues/2524
-    "ignore:The 'app' shortcut is now deprecated:DeprecationWarning"
+    "ignore:The 'app' shortcut is now deprecated:DeprecationWarning",
 ]
 xfail_strict = true
 
diff --git a/ragna/assistants/_ai21labs.py b/ragna/assistants/_ai21labs.py
index 289840b..19cfd59 100644
--- a/ragna/assistants/_ai21labs.py
+++ b/ragna/assistants/_ai21labs.py
@@ -1,6 +1,6 @@
 from typing import AsyncIterator, cast
 
-from ragna.core import RagnaException, Source
+from ragna.core import Source
 
 from ._api import ApiAssistant
 
@@ -8,17 +8,11 @@ from ._api import ApiAssistant
 class Ai21LabsAssistant(ApiAssistant):
     _API_KEY_ENV_VAR = "AI21_API_KEY"
     _MODEL_TYPE: str
-    _CONTEXT_SIZE: int = 8_192
-    # See https://docs.ai21.com/docs/model-availability-across-platforms#ai21-studio
 
     @classmethod
     def display_name(cls) -> str:
         return f"AI21Labs/jurassic-2-{cls._MODEL_TYPE}"
 
-    @property
-    def max_input_size(self) -> int:
-        return self._CONTEXT_SIZE
-
     def _make_system_content(self, sources: list[Source]) -> str:
         instruction = (
             "You are a helpful assistant that answers user questions given the context below. "
@@ -53,11 +47,7 @@ class Ai21LabsAssistant(ApiAssistant):
                 "system": self._make_system_content(sources),
             },
         )
-
-        if response.is_error:
-            raise RagnaException(
-                status_code=response.status_code, response=response.json()
-            )
+        await self._assert_api_call_is_success(response)
 
         yield cast(str, response.json()["outputs"][0]["text"])
 
diff --git a/ragna/assistants/_anthropic.py b/ragna/assistants/_anthropic.py
index 652bab7..de09e53 100644
--- a/ragna/assistants/_anthropic.py
+++ b/ragna/assistants/_anthropic.py
@@ -11,16 +11,11 @@ from ._api import ApiAssistant
 class AnthropicApiAssistant(ApiAssistant):
     _API_KEY_ENV_VAR = "ANTHROPIC_API_KEY"
     _MODEL: str
-    _CONTEXT_SIZE: int
 
     @classmethod
     def display_name(cls) -> str:
         return f"Anthropic/{cls._MODEL}"
 
-    @property
-    def max_input_size(self) -> int:
-        return self._CONTEXT_SIZE
-
     def _instructize_prompt(self, prompt: str, sources: list[Source]) -> str:
         # See https://docs.anthropic.com/claude/docs/introduction-to-prompt-design#human--assistant-formatting
         instruction = (
@@ -53,6 +48,8 @@ class AnthropicApiAssistant(ApiAssistant):
                 "stream": True,
             },
         ) as event_source:
+            await self._assert_api_call_is_success(event_source.response)
+
             async for sse in event_source.aiter_sse():
                 data = json.loads(sse.data)
                 if data["type"] != "completion":
@@ -74,7 +71,6 @@ class ClaudeInstant(AnthropicApiAssistant):
     """
 
     _MODEL = "claude-instant-1"
-    _CONTEXT_SIZE = 100_000
 
 
 class Claude(AnthropicApiAssistant):
@@ -86,4 +82,3 @@ class Claude(AnthropicApiAssistant):
     """
 
     _MODEL = "claude-2"
-    _CONTEXT_SIZE = 100_000
diff --git a/ragna/assistants/_api.py b/ragna/assistants/_api.py
index cc03650..1a5bd3a 100644
--- a/ragna/assistants/_api.py
+++ b/ragna/assistants/_api.py
@@ -1,11 +1,14 @@
 import abc
+import contextlib
+import json
 import os
 from typing import AsyncIterator
 
 import httpx
+from httpx import Response
 
 import ragna
-from ragna.core import Assistant, EnvVarRequirement, Requirement, Source
+from ragna.core import Assistant, EnvVarRequirement, RagnaException, Requirement, Source
 
 
 class ApiAssistant(Assistant):
@@ -35,3 +38,19 @@ class ApiAssistant(Assistant):
         self, prompt: str, sources: list[Source], *, max_new_tokens: int
     ) -> AsyncIterator[str]:
         ...
+
+    async def _assert_api_call_is_success(self, response: Response) -> None:
+        if response.is_success:
+            return
+
+        content = await response.aread()
+        with contextlib.suppress(Exception):
+            content = json.loads(content)
+
+        raise RagnaException(
+            "API call failed",
+            request_method=response.request.method,
+            request_url=str(response.request.url),
+            response_status_code=response.status_code,
+            response_content=content,
+        )
diff --git a/ragna/assistants/_cohere.py b/ragna/assistants/_cohere.py
index 7e5c928..a93a264 100644
--- a/ragna/assistants/_cohere.py
+++ b/ragna/assistants/_cohere.py
@@ -9,17 +9,11 @@ from ._api import ApiAssistant
 class CohereApiAssistant(ApiAssistant):
     _API_KEY_ENV_VAR = "COHERE_API_KEY"
     _MODEL: str
-    _CONTEXT_SIZE: int = 4_000
-    # See https://docs.cohere.com/docs/models#command
 
     @classmethod
     def display_name(cls) -> str:
         return f"Cohere/{cls._MODEL}"
 
-    @property
-    def max_input_size(self) -> int:
-        return self._CONTEXT_SIZE
-
     def _make_preamble(self) -> str:
         return (
             "You are a helpful assistant that answers user questions given the included context. "
@@ -54,12 +48,15 @@ class CohereApiAssistant(ApiAssistant):
                 "documents": self._make_source_documents(sources),
             },
         ) as response:
-            if response.is_error:
-                raise RagnaException(status_code=response.status_code)
+            await self._assert_api_call_is_success(response)
+
             async for chunk in response.aiter_lines():
                 event = json.loads(chunk)
                 if event["event_type"] == "stream-end":
-                    break
+                    if event["event_type"] == "COMPLETE":
+                        break
+
+                    raise RagnaException(event["error_message"])
                 if "text" in event:
                     yield cast(str, event["text"])
 
diff --git a/ragna/assistants/_demo.py b/ragna/assistants/_demo.py
index 5178bd1..e441b58 100644
--- a/ragna/assistants/_demo.py
+++ b/ragna/assistants/_demo.py
@@ -1,5 +1,4 @@
 import re
-import sys
 import textwrap
 from typing import Iterator
 
@@ -23,10 +22,6 @@ class RagnaDemoAssistant(Assistant):
     def display_name(cls) -> str:
         return "Ragna/DemoAssistant"
 
-    @property
-    def max_input_size(self) -> int:
-        return sys.maxsize
-
     def answer(self, prompt: str, sources: list[Source]) -> Iterator[str]:
         if re.search("markdown", prompt, re.IGNORECASE):
             yield self._markdown_answer()
diff --git a/ragna/assistants/_google.py b/ragna/assistants/_google.py
index d27df04..4de69ce 100644
--- a/ragna/assistants/_google.py
+++ b/ragna/assistants/_google.py
@@ -28,7 +28,6 @@ class AsyncIteratorReader:
 class GoogleApiAssistant(ApiAssistant):
     _API_KEY_ENV_VAR = "GOOGLE_API_KEY"
     _MODEL: str
-    _CONTEXT_SIZE: int
 
     @classmethod
     def requirements(cls) -> list[Requirement]:
@@ -41,10 +40,6 @@ class GoogleApiAssistant(ApiAssistant):
     def display_name(cls) -> str:
         return f"Google/{cls._MODEL}"
 
-    @property
-    def max_input_size(self) -> int:
-        return self._CONTEXT_SIZE
-
     def _instructize_prompt(self, prompt: str, sources: list[Source]) -> str:
         # https://ai.google.dev/docs/prompt_best_practices#add-contextual-information
         return "\n".join(
@@ -87,6 +82,8 @@ class GoogleApiAssistant(ApiAssistant):
                 },
             },
         ) as response:
+            await self._assert_api_call_is_success(response)
+
             async for chunk in ijson.items(
                 AsyncIteratorReader(response.aiter_bytes(1024)),
                 "item.candidates.item.content.parts.item.text",
@@ -107,7 +104,6 @@ class GeminiPro(GoogleApiAssistant):
     """
 
     _MODEL = "gemini-pro"
-    _CONTEXT_SIZE = 30_720
 
 
 class GeminiUltra(GoogleApiAssistant):
@@ -123,4 +119,3 @@ class GeminiUltra(GoogleApiAssistant):
     """
 
     _MODEL = "gemini-ultra"
-    _CONTEXT_SIZE = 30_720
diff --git a/ragna/assistants/_mosaicml.py b/ragna/assistants/_mosaicml.py
index 5234119..64edac1 100644
--- a/ragna/assistants/_mosaicml.py
+++ b/ragna/assistants/_mosaicml.py
@@ -1,6 +1,6 @@
 from typing import AsyncIterator, cast
 
-from ragna.core import RagnaException, Source
+from ragna.core import Source
 
 from ._api import ApiAssistant
 
@@ -8,16 +8,11 @@ from ._api import ApiAssistant
 class MosaicmlApiAssistant(ApiAssistant):
     _API_KEY_ENV_VAR = "MOSAICML_API_KEY"
     _MODEL: str
-    _CONTEXT_SIZE: int
 
     @classmethod
     def display_name(cls) -> str:
         return f"MosaicML/{cls._MODEL}"
 
-    @property
-    def max_input_size(self) -> int:
-        return self._CONTEXT_SIZE
-
     def _instructize_prompt(self, prompt: str, sources: list[Source]) -> str:
         # See https://huggingface.co/mosaicml/mpt-7b-instruct#formatting
         instruction = (
@@ -43,10 +38,8 @@ class MosaicmlApiAssistant(ApiAssistant):
                 "parameters": {"temperature": 0.0, "max_new_tokens": max_new_tokens},
             },
         )
-        if response.is_error:
-            raise RagnaException(
-                status_code=response.status_code, response=response.json()
-            )
+        await self._assert_api_call_is_success(response)
+
         yield cast(str, response.json()["outputs"][0]).replace(instruction, "").strip()
 
 
@@ -60,7 +53,6 @@ class Mpt7bInstruct(MosaicmlApiAssistant):
 
     # https://huggingface.co/mosaicml/mpt-7b-instruct#model-description
     _MODEL = "mpt-7b-instruct"
-    _CONTEXT_SIZE = 2048
 
 
 class Mpt30bInstruct(MosaicmlApiAssistant):
@@ -73,4 +65,3 @@ class Mpt30bInstruct(MosaicmlApiAssistant):
 
     # https://huggingface.co/mosaicml/mpt-30b-instruct#model-description
     _MODEL = "mpt-30b-instruct"
-    _CONTEXT_SIZE = 8_192
diff --git a/ragna/assistants/_openai.py b/ragna/assistants/_openai.py
index cfd4d5b..095ae9c 100644
--- a/ragna/assistants/_openai.py
+++ b/ragna/assistants/_openai.py
@@ -11,16 +11,11 @@ from ._api import ApiAssistant
 class OpenaiApiAssistant(ApiAssistant):
     _API_KEY_ENV_VAR = "OPENAI_API_KEY"
     _MODEL: str
-    _CONTEXT_SIZE: int
 
     @classmethod
     def display_name(cls) -> str:
         return f"OpenAI/{cls._MODEL}"
 
-    @property
-    def max_input_size(self) -> int:
-        return self._CONTEXT_SIZE
-
     def _make_system_content(self, sources: list[Source]) -> str:
         # See https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb
         instruction = (
@@ -60,6 +55,8 @@ class OpenaiApiAssistant(ApiAssistant):
                 "stream": True,
             },
         ) as event_source:
+            await self._assert_api_call_is_success(event_source.response)
+
             async for sse in event_source.aiter_sse():
                 data = json.loads(sse.data)
                 choice = data["choices"][0]
@@ -78,7 +75,6 @@ class Gpt35Turbo16k(OpenaiApiAssistant):
     """
 
     _MODEL = "gpt-3.5-turbo-16k"
-    _CONTEXT_SIZE = 16_384
 
 
 class Gpt4(OpenaiApiAssistant):
@@ -90,4 +86,3 @@ class Gpt4(OpenaiApiAssistant):
     """
 
     _MODEL = "gpt-4"
-    _CONTEXT_SIZE = 8_192
diff --git a/ragna/core/_components.py b/ragna/core/_components.py
index 5dad822..d237c1b 100644
--- a/ragna/core/_components.py
+++ b/ragna/core/_components.py
@@ -212,11 +212,6 @@ class Assistant(Component, abc.ABC):
 
     __ragna_protocol_methods__ = ["answer"]
 
-    @property
-    @abc.abstractmethod
-    def max_input_size(self) -> int:
-        ...
-
     @abc.abstractmethod
     def answer(self, prompt: str, sources: list[Source]) -> Iterator[str]:
         """Answer a prompt given some sources.
diff --git a/ragna/deploy/_ui/modal_configuration.py b/ragna/deploy/_ui/modal_configuration.py
index 7567c57..6821fa5 100644
--- a/ragna/deploy/_ui/modal_configuration.py
+++ b/ragna/deploy/_ui/modal_configuration.py
@@ -36,7 +36,7 @@ class ChatConfig(param.Parameterized):
     max_context_tokens = param.Integer(
         step=500,
         bounds=(1, 8000),
-        default=4000,
+        default=2000,
         doc=(
             "Maximum number of context tokens and in turn the number of document chunks "
             "pulled out of the vector database."


Test patch:
diff --git a/tests/assistants/__init__.py b/tests/assistants/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/assistants/test_api.py b/tests/assistants/test_api.py
new file mode 100644
index 0000000..9796145
--- /dev/null
+++ b/tests/assistants/test_api.py
@@ -0,0 +1,28 @@
+import os
+
+import pytest
+
+from ragna import assistants
+from ragna._compat import anext
+from ragna.assistants._api import ApiAssistant
+from ragna.core import RagnaException
+from tests.utils import skip_on_windows
+
+API_ASSISTANTS = [
+    assistant
+    for assistant in assistants.__dict__.values()
+    if isinstance(assistant, type)
+    and issubclass(assistant, ApiAssistant)
+    and assistant is not ApiAssistant
+]
+
+
+@skip_on_windows
+@pytest.mark.parametrize("assistant", API_ASSISTANTS)
+async def test_api_call_error_smoke(mocker, assistant):
+    mocker.patch.dict(os.environ, {assistant._API_KEY_ENV_VAR: "SENTINEL"})
+
+    chunks = assistant().answer(prompt="?", sources=[])
+
+    with pytest.raises(RagnaException, match="API call failed"):
+        await anext(chunks)
diff --git a/tests/deploy/api/test_e2e.py b/tests/deploy/api/test_e2e.py
index 278c776..9acebf0 100644
--- a/tests/deploy/api/test_e2e.py
+++ b/tests/deploy/api/test_e2e.py
@@ -4,25 +4,41 @@ import httpx_sse
 import pytest
 from fastapi.testclient import TestClient
 
+from ragna.assistants import RagnaDemoAssistant
 from ragna.deploy import Config
 from ragna.deploy._api import app
 
 from .utils import authenticate
 
 
-@pytest.mark.parametrize("stream_answer", [True, False])
-def test_e2e(tmp_local_root, stream_answer):
-    config = Config(local_root=tmp_local_root)
-    check_api(config, stream_answer=stream_answer)
+class TestAssistant(RagnaDemoAssistant):
+    def answer(self, prompt, sources, *, multiple_answer_chunks: bool):
+        content = next(super().answer(prompt, sources))
+
+        if multiple_answer_chunks:
+            for chunk in content.split(" "):
+                yield f"{chunk} "
+        else:
+            yield content
 
 
-def check_api(config, *, stream_answer):
+@pytest.mark.parametrize("multiple_answer_chunks", [True, False])
+@pytest.mark.parametrize("stream_answer", [True, False])
+def test_e2e(tmp_local_root, multiple_answer_chunks, stream_answer):
+    config = Config(local_root=tmp_local_root, assistants=[TestAssistant])
+
     document_root = config.local_root / "documents"
     document_root.mkdir()
     document_path = document_root / "test.txt"
     with open(document_path, "w") as file:
         file.write("!\n")
 
+    # Reset starlette_sse AppStatus for each run
+    # See https://github.com/sysid/sse-starlette/issues/59
+    from sse_starlette.sse import AppStatus
+
+    AppStatus.should_exit_event = None
+
     with TestClient(app(config=config, ignore_unavailable_components=False)) as client:
         authenticate(client)
 
@@ -66,7 +82,7 @@ def check_api(config, *, stream_answer):
             "name": "test-chat",
             "source_storage": source_storage,
             "assistant": assistant,
-            "params": {},
+            "params": {"multiple_answer_chunks": multiple_answer_chunks},
             "documents": [document],
         }
         chat = client.post("/chats", json=chat_metadata).raise_for_status().json()
@@ -96,10 +112,10 @@ def check_api(config, *, stream_answer):
                 json={"prompt": prompt, "stream": True},
             ) as event_source:
                 for sse in event_source.iter_sse():
-                    chunk = json.loads(sse.data)
-                    chunks.append(chunk["content"])
-            message = chunk
-            message["content"] = "".join(chunks)
+                    chunks.append(json.loads(sse.data))
+            message = chunks[0]
+            assert all(chunk["sources"] is None for chunk in chunks[1:])
+            message["content"] = "".join(chunk["content"] for chunk in chunks)
         else:
             message = (
                 client.post(f"/chats/{chat['id']}/answer", json={"prompt": prompt})
diff --git a/tests/utils.py b/tests/utils.py
new file mode 100644
index 0000000..356da1d
--- /dev/null
+++ b/tests/utils.py
@@ -0,0 +1,7 @@
+import platform
+
+import pytest
+
+skip_on_windows = pytest.mark.skipif(
+    platform.system() == "Windows", reason="Test is broken skipped on Windows"
+)