a

ape/llm.py

@@ -19,6 +19,8 @@ Cost = {
 
 
 class LLMGPT(object):
+    encoding = tiktoken.get_encoding("cl100k_base")
+
     def __init__(self, openai_key):
         self.gen_llm = ChatOpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, verbose=True)
         self.eval_llm = OpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, logprobs=1, verbose=True)
@@ -32,8 +34,8 @@ class LLMGPT(object):
             cost = 0.02
         else:
             cost = 0.0004
-        encoding = tiktoken.get_encoding("cl100k_base")
-        num_tokens = len(encoding.encode(text))
+
+        num_tokens = len(LLMGPT.encoding.encode(text))
         total_price = ((num_tokens / 1000) * cost)
         return total_price
 
@@ -78,12 +80,17 @@ class LLMGPT(object):
             eval_prompt = MyTemplate['eval_prompt']
         prompt = PromptTemplate.from_template(eval_prompt)
         eval_chain = LLMChain(llm=self.eval_llm, prompt=prompt)
-
+        score = 0
         for sample in eval_set:
+            output_len = len(LLMGPT.encoding.encode(sample[1]))
+            print(output_len)
             llmresult = eval_chain.generate([{'instruction': instruction, 'input': sample[0], 'output': sample[1]}])
             print(llmresult)
-            logprobs = llmresult.generations.[0][0].generation_info['token_logprobs']
+            logprobs = llmresult.generations[0][0].generation_info['logprobs']
             print(logprobs)
-            tokens = llmresult.generations.[0][0].generation_info['tokens']
+            tokens = logprobs['tokens']
+            token_probs = logprobs['token_logprobs']
             print(tokens)
-        return sum(logprobs)
+            print(token_probs)
+            score += sum(token_probs)
+        return score