Spaces:
Paused
Paused
File size: 976 Bytes
e4f691b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
from hackaprompt.completers import get_completer
import os
openai_api_key = os.getenv("OPENAI_API_KEY")
def test_count_tokens_gpt_35_turbo():
completer = get_completer(model="gpt-3.5-turbo", openai_api_key=openai_api_key)
prompt = "hello"
expected_token_count = 1 # taken on https://platform.openai.com/tokenizer
assert completer.get_token_count(prompt) == expected_token_count
prompt = "This is a test to see how many tokens we have"
expected_token_count = 11 # taken on https://platform.openai.com/tokenizer
assert completer.get_token_count(prompt) == expected_token_count
def test_count_tokens_flanT5():
completer = get_completer(model="FlanT5-XXL")
prompt = "hello"
expected_token_count = 2
assert completer.get_token_count(prompt) == expected_token_count
prompt = "This is a test to see how many tokens we have"
expected_token_count = 14
assert completer.get_token_count(prompt) == expected_token_count
|