a

ape/llm.py

@@ -80,7 +80,10 @@ class LLMGPT(object):
         eval_chain = LLMChain(llm=self.eval_llm, prompt=prompt)
 
         for sample in eval_set:
-            result = eval_chain.generate([{'instruction': instruction, 'input': sample[0], 'output': sample[1]}])
-            print(result)
-            logprobs = result[0][0].generation_info['logprobs']
+            llmresult = eval_chain.generate([{'instruction': instruction, 'input': sample[0], 'output': sample[1]}])
+            print(llmresult)
+            logprobs = llmresult.generations.[0][0].generation_info['token_logprobs']
             print(logprobs)
+            tokens = llmresult.generations.[0][0].generation_info['tokens']
+            print(tokens)
+        return sum(logprobs)