Spaces:
Runtime error
Runtime error
File size: 732 Bytes
58d33f0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
"""Test LLM utility functions."""
from langchain.llms.utils import enforce_stop_tokens
def test_enforce_stop_tokens() -> None:
"""Test removing stop tokens when they occur."""
text = "foo bar baz"
output = enforce_stop_tokens(text, ["moo", "baz"])
assert output == "foo bar "
text = "foo bar baz"
output = enforce_stop_tokens(text, ["moo", "baz", "bar"])
assert output == "foo "
text = "foo bar baz"
output = enforce_stop_tokens(text, ["moo", "bar"])
assert output == "foo "
def test_enforce_stop_tokens_none() -> None:
"""Test removing stop tokens when they do not occur."""
text = "foo bar baz"
output = enforce_stop_tokens(text, ["moo"])
assert output == "foo bar baz"
|