a

ape/ape.py

@@ -35,27 +35,28 @@ def esttimate_cost(instance):
     return f'Train={train_cost} Eval={eval_cost}'
 
 
-def generate(instance, openai_key, n_instruct):
+def generate(instance, openai_key):
     global LLM
     if LLM is None:
-        LLM = LLMGPT(openai_key, n_instruct)
+        LLM = LLMGPT(openai_key)
 
     instructions = []
     train_iter = instance.get_train_iter()
     for few_shot in train_iter:
         instruction = LLM.generate_instruction(few_shot)
         print(instruction)
-        instructions.append(instruction)
+        instructions.append(instruction['text'])
     return '\n'.join(instructions)
 
 
-def single_test(instruction, input, openai_key, n_instruct):
+def single_test(instruction, input, openai_key):
     global LLM
     if LLM is None:
-        LLM = LLMGPT(openai_key, n_instruct)
+        LLM = LLMGPT(openai_key)
 
     output = LLM.generate_output(instruction, input)
-    return output
+    return output['text']
 
-def score_single(instruction, openai_key, n_instruct):
+
+def score_single(instruction, openai_key):
     pass

ape/instance.py

@@ -33,7 +33,7 @@ class Instance(object):
         index = random.sample(list(range(len(self.samples))), n_train + n_eval)
         train_index, eval_index = index[:n_train], index[n_train:]
         self.train_samples = [self.samples[i] for i in train_index]
-        self.eval_samples = [self.samples[i] for i in train_index]
+        self.eval_samples = [self.samples[i] for i in eval_index]
 
     def get_train_iter(self):
         for i in range(self.n_train):

ape/llm.py

@@ -19,8 +19,8 @@ Cost = {
 
 
 class LLMGPT(object):
-    def __init__(self, openai_key, n_instruct):
-        self.gen_llm = ChatOpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, n=n_instruct, verbose=True)
+    def __init__(self, openai_key):
+        self.gen_llm = ChatOpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, verbose=True)
         self.eval_llm = OpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, echo=True, verbose=True)
         self.test_llm = ChatOpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, verbose=True)
         self.gen_chain = None

ape/prompt.py

@@ -2,7 +2,9 @@
 few_shot_prompt = "Input: {input}\nOutput: {output}\n"
 
 gen_user_prompt = """I gave a friend who is medical school teacher an instruction. 
-Based on the instruction they produced the following input-output pairs:\n\n{few_shot}\n\nThe instruction was to """
+Based on the instruction they produced the following input-output pairs:\n\n{few_shot}\n\n. What instruction can best
+explain the relationship between above input-output pairs? Given me a concise instruction in chinese: 
+"""
 
 gen_sys_prompt = ""
 
@@ -10,8 +12,8 @@ eval_prompt = "Instruction: {instruction}\nInput: {input}\nOutput: {output}"
 
 test_sys_prompt = ""
 
-test_user_prompt = """I gave a friend who is medical school teacher an instruction. Base on this instruction, they will
-transform the input to output.\n Instruction: {instruction}Input: {input}\nOutput: """
+test_user_prompt = """I gave a friend who is medical school teacher an instruction. They followed the instruction and 
+transform the input to output in chinese.\n Instruction: {instruction}\nInput: {input}\nOutput: """
 
 MyTemplate = {
     'gen_user_prompt': gen_user_prompt,

app.py

@@ -19,7 +19,7 @@ with gr.Blocks(title="Automatic Prompt Engineer", theme=gr.themes.Glass()) as de
 
             with gr.Row():
                 n_eval = gr.Slider(label="Number of Eval", minimum=5, maximum=30, step=5, value=20)
-                n_instruct = gr.Slider(label="Number of Prompt", minimum=1, maximum=5, step=1, value=2)
+
 
         with gr.Column(scale=3):
             gr.Markdown("## Load Data")
@@ -44,7 +44,7 @@ with gr.Blocks(title="Automatic Prompt Engineer", theme=gr.themes.Glass()) as de
         with gr.Column(scale=2):
             gr.Markdown("## Run APE")
             gen_prompt = gr.Textbox(max_lines=100, lines=10,
-                                    value=MyTemplate['gen_sys_prompt'], label="Prompt for generation")
+                                    value=MyTemplate['gen_user_prompt'], label="Prompt for generation")
             eval_prompt = gr.Textbox(max_lines=100, lines=10,
                                      value=MyTemplate['eval_prompt'], label="Prompt for Evaluation")
 
@@ -94,14 +94,14 @@ with gr.Blocks(title="Automatic Prompt Engineer", theme=gr.themes.Glass()) as de
     cost_button.click(esttimate_cost, [instance], [cost])
 
     # 4. Run APE -> 所有指令
-    ape_button.click(generate, [instance, openai_key, n_instruct], [all_prompt])
+    ape_button.click(generate, [instance, openai_key], [all_prompt])
 
     # 5. Evaluate -> 得到所有指令的Log Prob
 
     # 6. 输入指令单测
-    test_button.click(single_test, [test_prompt, test_input, openai_key, n_instruct], [test_output])
+    test_button.click(single_test, [test_prompt, test_input, openai_key], [test_output])
 
     # 7. 输入指令打分
-    score_button.click(score_single, [score_prompt, openai_key, n_instruct], [test_score])
+    score_button.click(score_single, [score_prompt, openai_key], [test_score])
 
     demo.launch(show_error=True)