Repository: lynxius/lynxius-python (language: python)

Issue:
Added PyYAML and numpy as testing requirements



Discussion / hints:

Fix patch:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 252d769..08befb9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -38,7 +38,7 @@ Leave an empty line to separate each import group.
 
 ## Lint And Format Your Code
 
-We reccomand setting up `flake8` and `black` to style your Python code.
+We reccomend setting up `flake8` and `black` to style your Python code.
 
 ### Setup Flake8 And Black on VSCode
 
diff --git a/requirements.txt b/requirements.txt
index 839bc7f..2012b9d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -90,6 +90,7 @@ pydantic==2.7.4
 pydantic_core==2.18.4
 Pygments==2.17.2
 pyproject_hooks==1.0.0
+pytest==8.3.2
 python-dateutil==2.9.0.post0
 python-json-logger==2.0.7
 PyYAML==6.0.1
diff --git a/setup.py b/setup.py
index d8cb3c0..3fc7c09 100644
--- a/setup.py
+++ b/setup.py
@@ -16,9 +16,7 @@
     "openai>=1.35.1",
 ]
 
-test_requirements = [
-    "pytest>=3",
-]
+test_requirements = ["pytest>=3", "PyYAML>=6.0.1", "numpy>=2.0.0"]
 
 setup(
     author="Lynxius Inc.",
@@ -51,6 +49,9 @@
     ),
     test_suite="tests",
     tests_require=test_requirements,
+    extras_require={
+        "test": test_requirements,
+    },
     url="https://github.com/lynxius/lynxius-python",
     version="1.2.0",
     zip_safe=False,


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..1230852
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,34 @@
+name: Run Tests
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+
+      - name: Set LYNXIUS_API_KEY environment variable
+        env:
+          LYNXIUS_API_KEY: ${{ secrets.LYNXIUS_AUTOMATED_TESTING_KEY }}
+        run: echo "Environment variable LYNXIUS_API_KEY is set"
+
+      - name: Run tests
+        run: pytest
+        env:
+          LYNXIUS_API_KEY: ${{ secrets.LYNXIUS_AUTOMATED_TESTING_KEY }}
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/test/test_context_precision.py b/test/test_context_precision.py
new file mode 100644
index 0000000..f565994
--- /dev/null
+++ b/test/test_context_precision.py
@@ -0,0 +1,114 @@
+import os
+import pytest
+import yaml
+import numpy as np
+from lynxius.client import LynxiusClient
+from lynxius.evals.context_precision import ContextPrecision
+from lynxius.rag.types import ContextChunk
+
+THRESHOLD = 0.15
+api_key = os.getenv('LYNXIUS_API_KEY')
+
+
+@pytest.fixture(scope="class")
+def yaml_data(request):
+    relative_path = os.path.join(
+        os.path.dirname(__file__),
+        "test_context_precision.yaml",
+    )
+    TEST_FILE_PATH = os.path.normpath(relative_path)
+
+    # Open the YAML file
+    with open(TEST_FILE_PATH, "r") as file:
+        data = yaml.safe_load(file)
+
+    # Attach the data to the class under test
+    if request.cls is not None:
+        request.cls.yaml_data = data
+
+    yield
+
+
+@pytest.fixture(scope="class")
+def init_client():
+    return LynxiusClient(api_key=api_key)
+
+
+@pytest.fixture(scope="class")
+def init_model():
+    label = "iter_1"
+    tags = ["context_precision"]
+    return ContextPrecision(label=label, tags=tags)
+
+
+def calculate_statistics(scores):
+    # Calculate average
+    average_score = np.mean(scores)
+
+    # Calculate 20th and 90th percentiles
+    percentile_20 = np.percentile(scores, 20, method="inverted_cdf")
+    percentile_90 = np.percentile(scores, 90, method="inverted_cdf")
+
+    return average_score, percentile_20, percentile_90
+
+
+@pytest.mark.usefixtures("yaml_data")
+class TestContextPrecision:
+    """Test `ContextPrecision` evaluator."""
+
+    @pytest.fixture(scope="class")
+    def run_eval(self, init_client, init_model):
+        scores = []
+        for entry in self.yaml_data['test_cases']:
+            init_model.add_trace(
+                query=entry["input"]["query"],
+                reference=entry["input"]["reference"],
+                context=[ContextChunk(document=ctx["document"], relevance=ctx["relevance"]) for ctx in entry["input"]["contexts"]]
+            )
+
+        eval_run_id = init_client.evaluate(init_model)
+        eval_run = init_client.get_eval_run(eval_run_id)
+        scores.extend([float(result.get("score")) for result in eval_run["results"]])
+        
+        eval_run["statistics"] = calculate_statistics(scores)
+        return eval_run
+
+    def test_init(self, init_model):
+        assert init_model.label == "iter_1"
+        assert "context_precision" in init_model.tags
+
+    def test_yaml_data(self):
+        # Access the YAML data attached to the class
+        assert len(self.yaml_data['test_cases']) == 18
+
+    def test_evaluate_high_score(self, run_eval):
+        entry = self.yaml_data['test_cases'][0]
+        expected_score = float(entry["expected_output"])
+
+        result_score = float(run_eval["results"][0].get("score"))
+        assert abs(result_score - expected_score) <= THRESHOLD
+
+    def test_evaluate_low_score(self, run_eval):
+        entry = self.yaml_data['test_cases'][1]
+        expected_score = float(entry["expected_output"])
+
+        result_score = float(run_eval["results"][1].get("score"))
+        assert abs(result_score - expected_score) <= THRESHOLD
+
+    def test_evaluate_medium_score(self, run_eval):
+        entry = self.yaml_data['test_cases'][2]
+        expected_score = float(entry["expected_output"])
+
+        result_score = float(run_eval["results"][2].get("score"))
+        assert abs(result_score - expected_score) <= THRESHOLD
+
+    def test_statistics(self, run_eval):
+        expected_average_score, expected_percentile_20, expected_percentile_90 = run_eval["statistics"]
+
+        aggregate_score = float(run_eval.get("aggregate_score"))
+        p20 = float(run_eval.get("p20"))
+        p90 = float(run_eval.get("p90"))
+
+        assert abs(expected_average_score - aggregate_score) <= THRESHOLD
+        assert abs(expected_percentile_20 - p20) <= THRESHOLD
+        assert abs(expected_percentile_90 - p90) <= THRESHOLD
\ No newline at end of file
diff --git a/test/test_context_precision.yaml b/test/test_context_precision.yaml
new file mode 100644
index 0000000..81b66c8
--- /dev/null
+++ b/test/test_context_precision.yaml
@@ -0,0 +1,39 @@
+test_cases:
+  # Success cases (expected output 1.0)
+  - input:
+      query: "What is the capital of France?"
+      reference: "The capital of France is Paris."
+      contexts:
+        - document: "Paris is the capital of France."
+          relevance: 1.0
+        - document: "France is a country in Europe."
+          relevance: 0.5
+        - document: "Berlin is the capital of Germany."
+          relevance: -1.0
+    expected_output: 1.0
+
+  # Failure cases (expected output 0.0)
+  - input:
+      query: "What is the capital of France?"
+      reference: "The capital of France is Paris."
+      contexts:
+        - document: "Tokyo is the capital of Japan."
+          relevance: -1.0
+        - document: "Japan is an island nation in Asia."
+          relevance: -1.0
+        - document: "The Great Wall of China is a historical landmark."
+          relevance: -1.0
+    expected_output: 0.0
+
+  # Partial success cases (expected output 0.5)
+  - input:
+      query: "What is the primary language spoken in Spain?"
+      reference: "The primary language spoken in Spain is Spanish."
+      contexts:
+        - document: "Spanish is the primary language in Spain."
+          relevance: 1.0
+        - document: "Spain is a country in Europe."
+          relevance: 0.5
+        - document: "The capital of Spain is Madrid."
+          relevance: 0.0
+    expected_output: 0.5
\ No newline at end of file
diff --git a/tests/answer_correctness_data.yaml b/tests/answer_correctness_data.yaml
new file mode 100644
index 0000000..4160beb
--- /dev/null
+++ b/tests/answer_correctness_data.yaml
@@ -0,0 +1,16 @@
+- query: "On what date was the Declaration of Independence officially signed? Keep it short."
+  reference: "The Declaration of Independence was signed on August 2, 1776."
+  output: "The Declaration of Independence was signed on August 2, 1776."
+  comments: "answered by gpt4o"
+  score: 1.0
+- query: "What personality do Libras have? Keep it short."
+  reference: "Astrological signs are not correlated with personality traits."
+  output: |
+    Libras are known for being charming, diplomatic, and social. They value balance, harmony, and fairness, often striving to create peaceful environments. Libras are also indecisive and can be people-pleasers, but their natural charm and ability to see multiple perspectives make them excellent mediators.
+  comments: "answered by gpt4o"
+  score: 0.0
+- query: "Where did fortune cookies originate? Keep it short."
+  reference: "The exact origin of fortune cookies is unclear, though various immigrant groups in California claim to have popularized them in the early 20th century. They most likely originated from cookies made by Japanese immigrants to the United States in the late 19th or early 20th century."
+  output: "Fortune cookies originated in the United States likely in California during the early 20th century. They were inspired by Japanese senbei crackers and popularized by Chinese-American restaurants."
+  comments: "answered by gpt4o, reference was added manually from wikipedia by me"
+  score: 0.5
\ No newline at end of file
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..355d786
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,5 @@
+import os
+import sys
+
+# Add the module path to sys.path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
diff --git a/tests/test_answer_correctness.py b/tests/test_answer_correctness.py
new file mode 100644
index 0000000..3f1c364
--- /dev/null
+++ b/tests/test_answer_correctness.py
@@ -0,0 +1,115 @@
+import os
+
+import numpy as np
+import pytest
+import yaml
+
+from lynxius.client import LynxiusClient
+from lynxius.evals.answer_correctness import AnswerCorrectness
+
+THRESHOLD = 0.1
+scores = []
+api_key = os.getenv("LYNXIUS_API_KEY")
+
+
+@pytest.fixture(scope="class")
+def yaml_data(request):
+    relative_path = os.path.join(
+        os.path.dirname(__file__),
+        "answer_correctness_data.yaml",
+    )
+    TEST_FILE_PATH = os.path.normpath(relative_path)
+
+    # Open the YAML file
+    with open(TEST_FILE_PATH, "r") as file:
+        data = yaml.safe_load(file)
+
+    # Attach the data to the class under test
+    if request.cls is not None:
+        request.cls.yaml_data = data
+
+    yield
+
+
+def calculate_statistics(scores):
+    # Calculate average
+    average_score = np.mean(scores)
+
+    # Calculate 20th and 90th percentiles
+    percentile_20 = np.percentile(scores, 20, method="inverted_cdf")
+    percentile_90 = np.percentile(scores, 90, method="inverted_cdf")
+
+    return average_score, percentile_20, percentile_90
+
+
+@pytest.mark.usefixtures("yaml_data")
+class TestAnswerCorrectness:
+    """Test `AnswerCorrectness` evaluator."""
+
+    @pytest.fixture(scope="class")
+    def init_client(self):
+        return LynxiusClient(api_key=api_key)
+
+    @pytest.fixture(scope="class")
+    def run_eval(self, init_client):
+        label = "unit_test"
+        tags = ["answer_correctness"]
+        eval = AnswerCorrectness(label=label, tags=tags)
+
+        for entry in self.yaml_data:
+            eval.add_trace(
+                query=entry["query"],
+                reference=entry["reference"],
+                output=entry["output"],
+            )
+
+        answer_correctness_uuid = init_client.evaluate(eval)
+        eval_run = init_client.get_eval_run(answer_correctness_uuid)
+
+        scores.extend([float(entry.get("score")) for entry in eval_run["results"]])
+
+        return eval_run
+
+    def test_init(self, run_eval):
+        assert run_eval["label"] == "unit_test"
+        assert "answer_correctness" in run_eval["tags"]
+
+    def test_statistics(self, run_eval):
+        expected_average_score, expected_percentile_20, expected_percentile_90 = (
+            calculate_statistics(scores)
+        )
+
+        aggregate_score = float(run_eval.get("aggregate_score"))
+        p20 = float(run_eval.get("p20"))
+        p90 = float(run_eval.get("p90"))
+
+        print(expected_average_score, expected_percentile_20, expected_percentile_90)
+        print(aggregate_score, p20, p90)
+
+        assert abs(expected_average_score - aggregate_score) <= THRESHOLD
+        assert abs(expected_percentile_20 - p20) <= THRESHOLD
+        assert abs(expected_percentile_90 - p90) <= THRESHOLD
+
+    def test_evaluate_high_score(self, run_eval):
+        entry = self.yaml_data[0]
+        expected_score = float(entry["score"])
+
+        score = float(run_eval["results"][0].get("score"))
+
+        assert abs(score - expected_score) <= THRESHOLD
+
+    def test_evaluate_low_score(self, run_eval):
+        entry = self.yaml_data[1]
+        expected_score = float(entry["score"])
+
+        score = float(run_eval["results"][1].get("score"))
+
+        assert abs(score - expected_score) <= THRESHOLD
+
+    def test_evaluate_medium_score(self, run_eval):
+        entry = self.yaml_data[2]
+        expected_score = float(entry["score"])
+
+        score = float(run_eval["results"][2].get("score"))
+
+        assert abs(score - expected_score) <= THRESHOLD