Chris4K commited on
Commit
b28599f
1 Parent(s): 8f5eeb9

Update token_counter.py

Browse files
Files changed (1) hide show
  1. token_counter.py +1 -1
token_counter.py CHANGED
@@ -10,6 +10,6 @@ class TokenCounterTool(Tool):
10
  def __call__(self, prompt: str):
11
  # token = os.environ['hf']
12
  tokenizer = AutoTokenizer.from_pretrained("lgaalves/gpt2-dolly")
13
- tokens = tokenizer(input_text)["input_ids"]
14
  return f"{len(tokens)}"
15
 
 
10
  def __call__(self, prompt: str):
11
  # token = os.environ['hf']
12
  tokenizer = AutoTokenizer.from_pretrained("lgaalves/gpt2-dolly")
13
+ tokens = tokenizer(prompt)["input_ids"]
14
  return f"{len(tokens)}"
15