Repository: protectai/llm-guard

Issue:
binary incompatibility when running the Gemini example: Expected 96 from C header, got 88 from PyObject
**Describe the bug**

I've run the code in https://github.com/protectai/llm-guard/blob/main/examples/google_gemini.py in Google Colab

**To Reproduce**
Steps to reproduce the behavior:
1. Create a python notebook with two pieces of code:
2. first from here: 
```bash
!pip install google-cloud-aiplatform --upgrade --user
!pip install llm-guard
```
4. second from here: Go to https://github.com/protectai/llm-guard/blob/main/examples/google_gemini.py
5. Put reasonable Project/region
6. Execute first, reload env, execute second.
7. See error

**Expected behavior**
Something good should happen - you tell me :)

**Screenshots**
If applicable, add screenshots to help explain your problem.
Coming in next comment

**Additional context**

See below


## first error
First it said It was compiled with different version of numpy, so I've added this as suggested by Gemini:

```
!pip install --upgrade numpy
```

## second error

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-1-d5c52cca659e> in <cell line: 30>()
     28 
     29 # Import Scanners from LLM Guard
---> 30 from llm_guard import scan_output, scan_prompt
     31 from llm_guard.input_scanners import PromptInjection, TokenLimit, Toxicity
     32 from llm_guard.output_scanners import NoRefusal, Relevance, Sensitive

14 frames
/usr/local/lib/python3.10/dist-packages/thinc/backends/numpy_ops.pyx in init thinc.backends.numpy_ops()

ValueError: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject
```

Discussion:
@palladius: ![image](https://github.com/user-attachments/assets/3b2fc0dc-739d-4198-b7e0-e0f5c60a3f11)
@palladius: First error - hopefully fixed by updating numpy (maybe exacerbated?)

![image](https://github.com/user-attachments/assets/e07860df-f55a-49b2-9a82-e42d44e69f62)
@asofter: Hey @palladius ! Thanks for reporting this! The issue has been addressed in version 0.3.16, which is now available on PyPI. Please upgrade and let us know if you encounter any further problems.



Solution patch:
diff --git a/docs/changelog.md b/docs/changelog.md
index d37fddd..bebf7cd 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -11,9 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -
 
 ### Fixed
--
+- Regex scanner redacts only the first occurrence ([#229](https://github.com/protectai/llm-guard/issues/229)).
+- BanSubstrings scanner redacts only the first occurrence ([#210](https://github.com/protectai/llm-guard/issues/210)).
 
 ### Changed
+- Upgrade all dependencies.
 - Stop substrings moved to the variables instead of JSON files.
 - **[BREAKING]** New logic to calculate the risk score ([#182](https://github.com/protectai/llm-guard/issues/182)).
 
diff --git a/examples/amazon_bedrock.py b/examples/amazon_bedrock.py
index 1c070f5..67becb5 100644
--- a/examples/amazon_bedrock.py
+++ b/examples/amazon_bedrock.py
@@ -68,7 +68,7 @@ response_text = invoke_agent(sanitized_prompt, REGION, AGENT_ID, AGENT_ALIAS_ID,
 sanitized_response_text, results_valid, results_score = scan_output(
     OUTPUT_SCANNERS, sanitized_prompt, response_text
 )
-if any(results_valid.values()) is False:
+if not all(results_valid.values()) is True:
     print(f"Output {response_text} is not valid, scores: {results_score}")
     exit(1)
 
diff --git a/examples/openai_api.py b/examples/openai_api.py
index 1d62c19..6868d84 100644
--- a/examples/openai_api.py
+++ b/examples/openai_api.py
@@ -45,7 +45,7 @@ response_text = response.choices[0].message.content
 sanitized_response_text, results_valid, results_score = scan_output(
     output_scanners, sanitized_prompt, response_text
 )
-if any(results_valid.values()) is False:
+if not all(results_valid.values()) is True:
     print(f"Output {response_text} is not valid, scores: {results_score}")
     exit(1)
 
diff --git a/llm_guard/input_scanners/ban_substrings.py b/llm_guard/input_scanners/ban_substrings.py
index 7ab1be2..c992734 100644
--- a/llm_guard/input_scanners/ban_substrings.py
+++ b/llm_guard/input_scanners/ban_substrings.py
@@ -87,11 +87,12 @@ class BanSubstrings(Scanner):
         self._redact = redact
         self._contains_all = contains_all
 
-    @staticmethod
-    def _redact_text(text: str, substrings: list[str]) -> str:
+    def _redact_text(self, text: str, substrings: list[str]) -> str:
         redacted_text = text
         for s in substrings:
-            redacted_text = redacted_text.replace(s, "[REDACTED]")
+            regex_redacted = re.compile(re.escape(s), 0 if self._case_sensitive else re.IGNORECASE)
+            redacted_text = regex_redacted.sub("[REDACTED]", redacted_text)
+
         return redacted_text
 
     def scan(self, prompt: str) -> tuple[str, bool, float]:
diff --git a/llm_guard/input_scanners/regex.py b/llm_guard/input_scanners/regex.py
index 1055fcd..28b7368 100644
--- a/llm_guard/input_scanners/regex.py
+++ b/llm_guard/input_scanners/regex.py
@@ -1,8 +1,6 @@
-from __future__ import annotations
-
 import re
 from enum import Enum
-from typing import Any, Pattern
+from typing import List, Pattern
 
 from presidio_anonymizer.core.text_replace_builder import TextReplaceBuilder
 
@@ -16,9 +14,23 @@ LOGGER = get_logger()
 class MatchType(Enum):
     SEARCH = "search"
     FULL_MATCH = "fullmatch"
+    ALL = "all"
+
+    def match(self, pattern: Pattern[str], text: str) -> List[re.Match[str]]:
+        if self.value == "all":
+            return list(pattern.finditer(text))[::-1]  # Reverse order to avoid index issues
+
+        m = None
+        if self.value == "search":
+            m = pattern.search(text)
+
+        if self.value == "fullmatch":
+            m = pattern.fullmatch(text)
+
+        if m is None:
+            return []
 
-    def match(self, pattern: Pattern[str], text: str) -> Any:
-        return getattr(pattern, self.value)(text)
+        return [m]
 
 
 class Regex(Scanner):
@@ -34,7 +46,7 @@ class Regex(Scanner):
         patterns: list[str],
         *,
         is_blocked: bool = True,
-        match_type: MatchType | str = MatchType.SEARCH,
+        match_type: MatchType | str = MatchType.ALL,
         redact: bool = True,
     ) -> None:
         """
@@ -43,11 +55,11 @@ class Regex(Scanner):
         Parameters:
             patterns (Sequence[str]): A list of regular expressions to use for pattern matching.
             is_blocked (bool): Whether the patterns are blocked or allowed.
-            match_type (str): The type of match to use. Can be either "search" or "fullmatch".
+            match_type (str): The type of match to use.
             redact (bool): Whether to redact the output or not.
 
         Raises:
-            ValueError: If no patterns provided or both good and bad patterns provided.
+            ValueError: If no patterns are provided or both good and bad patterns are provided.
         """
         if isinstance(match_type, str):
             match_type = MatchType(match_type)
@@ -63,19 +75,20 @@ class Regex(Scanner):
     def scan(self, prompt: str) -> tuple[str, bool, float]:
         text_replace_builder = TextReplaceBuilder(original_text=prompt)
         for pattern in self._patterns:
-            match = self._match_type.match(pattern, prompt)
-            if match is None:
+            matches = self._match_type.match(pattern, prompt)
+            if matches is None or len(matches) == 0:
                 continue
 
             if self._is_blocked:
                 LOGGER.warning("Pattern was detected in the text", pattern=pattern)
 
                 if self._redact:
-                    text_replace_builder.replace_text_get_insertion_index(
-                        "[REDACTED]",
-                        match.start(),
-                        match.end(),
-                    )
+                    for match in matches:
+                        text_replace_builder.replace_text_get_insertion_index(
+                            "[REDACTED]",
+                            match.start(),
+                            match.end(),
+                        )
 
                 return text_replace_builder.output_text, False, 1.0
 


Test patch:
diff --git a/tests/input_scanners/test_anonymize.py b/tests/input_scanners/test_anonymize.py
index 9bb0bc9..5e59076 100644
--- a/tests/input_scanners/test_anonymize.py
+++ b/tests/input_scanners/test_anonymize.py
@@ -226,6 +226,13 @@ Create a summarized version of his resume.""",
             False,
             1.0,
         ),
+        (
+            {"allowed_names": "Mary"},
+            "Full Name: Mary.",
+            "Full Name: Mary.",
+            True,
+            -1.0,
+        ),  # Exposed name
         ({}, "", "", True, -1.0),  # Empty prompt
     ],
 )
diff --git a/tests/input_scanners/test_ban_substrings.py b/tests/input_scanners/test_ban_substrings.py
index 311a41d..a0760cd 100644
--- a/tests/input_scanners/test_ban_substrings.py
+++ b/tests/input_scanners/test_ban_substrings.py
@@ -43,9 +43,12 @@ def test_scan_all():
     scanner = BanSubstrings(
         substrings=["one", "two"],
         redact=True,
+        case_sensitive=False,
         contains_all=True,
     )
-    sanitized_prompt, valid, score = scanner.scan("Testing one and two")
-    assert sanitized_prompt == "Testing [REDACTED] and [REDACTED]"
+    sanitized_prompt, valid, score = scanner.scan("Testing one and two. But also two and one.")
+    assert (
+        sanitized_prompt == "Testing [REDACTED] and [REDACTED]. But also [REDACTED] and [REDACTED]."
+    )
     assert valid is False
     assert score == 1.0
diff --git a/tests/input_scanners/test_regex.py b/tests/input_scanners/test_regex.py
index f9f7f6e..e659cfe 100644
--- a/tests/input_scanners/test_regex.py
+++ b/tests/input_scanners/test_regex.py
@@ -13,12 +13,18 @@ from llm_guard.input_scanners.regex import MatchType, Regex
             False,
             1.0,
         ),  # Regex matched
+        (
+            "What is 12345? 12345",
+            "What is [REDACTED]? [REDACTED]",
+            False,
+            1.0,
+        ),  # Multiple matches
     ],
 )
 def test_scan(prompt, expected_prompt, expected_valid, expected_score):
     scanner = Regex(
-        patterns=[r"Bearer [A-Za-z0-9-._~+/]+"],
-        match_type=MatchType.SEARCH,
+        patterns=[r"Bearer [A-Za-z0-9-._~+/]+", r"\d{5}"],
+        match_type=MatchType.ALL,
         is_blocked=True,
         redact=True,
     )
diff --git a/tests/test_evaluate.py b/tests/test_evaluate.py
index d83db79..c573052 100644
--- a/tests/test_evaluate.py
+++ b/tests/test_evaluate.py
@@ -3,7 +3,16 @@ import pytest
 from llm_guard import scan_output, scan_prompt
 from llm_guard.input_scanners import BanSubstrings, TokenLimit
 from llm_guard.input_scanners.ban_substrings import MatchType as BanSubstringsMatchType
-from llm_guard.output_scanners import BanTopics, NoRefusal, Regex, Relevance
+from llm_guard.output_scanners import (
+    BanTopics,
+    Deanonymize,
+    Gibberish,
+    NoRefusal,
+    Regex,
+    Relevance,
+    Sensitive,
+)
+from llm_guard.vault import Vault
 
 
 @pytest.mark.parametrize(
@@ -70,6 +79,20 @@ def test_scan_prompt(evaluators, prompt, expected_sanitized_prompt, expected_res
             {"Regex": False},
             True,
         ),  # fail fast
+        (
+            [Deanonymize(Vault()), NoRefusal(), Relevance(), Sensitive(), Gibberish()],
+            "test phrase",
+            "test output",
+            "test output",
+            {
+                "Deanonymize": True,
+                "NoRefusal": True,
+                "Relevance": True,
+                "Sensitive": True,
+                "Gibberish": True,
+            },
+            False,
+        ),  # multiple scanners
     ],
 )
 def test_scan_output(