a

ape/ape.py

@@ -1,24 +1,48 @@
 # -*-coding:utf-8 -*-
 
-from ape.instance import Instance, LoadFactory
+from ape.instance import Instance, LoadFactory, upload_file
+from ape.llm import LLMGPT
 from functools import partial
+from itertools import chain
+LLM = None
 
 
 def load_task(task, file):
     global instance
     if task:
         loader = LoadFactory[task]
-        print(loader)
     else:
+        file = [i.name for i in file]
         print(file)
-        loader = partial(LoadFactory['upload'], file=[i.name for i in file][0])
+        loader = partial(upload_file, file=file[0])
     instance = Instance.from_file(loader)
     print(instance.samples[0])
-    return instance
+    return instance, f'{instance.n_sample} Data Loaded'
 
 
 def sample_data(instance, n_train, n_few_shot, n_eval):
     instance.sample(n_train, n_few_shot, n_eval)
     train_str = instance.display(instance.train_samples)
     eval_str = instance.display(instance.eval_samples)
-    return train_str, eval_str, instance
+    return train_str, eval_str, instance, 'Sample Done'
+
+
+def esttimate_cost(instance):
+    train_text = ''.join(chain(*instance.train_samples))
+    eval_text = ''.join(chain(*instance.eval_samples))
+    train_cost = LLMGPT.confirm_cost(train_text, 'train')
+    eval_cost = LLMGPT.confirm_cost(eval_text, 'eval')
+    return f'Train={train_cost} Eval={eval_cost}'
+
+
+def generate(instance, openai_key, n_instruct):
+    global LLM
+    if LLM is None:
+        LLM = LLMGPT(openai_key, n_instruct)
+
+    instructions = []
+    train_iter = instance.get_train_iter()
+    for few_shot in train_iter:
+        instruction = LLM.generate_instruction(few_shot)
+        print(instruction)
+    return '\n'.join(instructions)

ape/instance.py

@@ -11,10 +11,21 @@ class Instance(object):
 
     def __init__(self, loader=None):
         self.samples = loader()
+        self.n_few_shot = 0
+        self.n_train = 0
+        self.n_eval = 0
+        self.train_iter = None
         self.train_samples = []
         self.eval_samples = []
 
+    @property
+    def n_sample(self):
+        return len(self.samples)
+
     def sample(self, n_train, n_few_shot, n_eval):
+        self.n_train = n_train
+        self.n_few_shot = n_few_shot
+        self.n_eval = n_eval
         n_train = n_train * n_few_shot
         if n_train + n_eval > len(self.samples):
             raise ValueError(f'Train + Eval > total samples {len(self.samples)}, decrease them')
@@ -24,6 +35,10 @@ class Instance(object):
         self.train_samples = [self.samples[i] for i in train_index]
         self.eval_samples = [self.samples[i] for i in train_index]
 
+    def get_train_iter(self):
+        for i in range(self.n_train):
+            yield self.train_samples[(i*self.n_few_shot) :(i+1)* self.n_few_shot]
+
     @staticmethod
     def display(samples):
         s = ""
@@ -80,8 +95,7 @@ def upload_file(file):
 LoadFactory = {
     'paraphase': load_paraphase,
     'event_extract': load_event_extraction,
-    'search_intent': load_intent,
-    'upload': upload_file
+    'search_intent': load_intent
 }
 
 if __name__ == '__main__':
@@ -100,3 +114,5 @@ if __name__ == '__main__':
                                     ('deliberately', 'accidentally'), ('off', 'on')])
     instance2.sample(n_train, few_shot, n_eval)
     print(instance2.display(instance2.train_samples))
+    train_iter = instance2.get_train_iter()
+    print(next(train_iter))

ape/llm.py

@@ -19,9 +19,9 @@ Cost = {
 
 
 class LLMGPT(object):
-    def __init__(self, openai_key, max_tokens, n_instruct):
-        self.gen_llm = ChatOpenAI(openai_api_key=openai_key, max_tokens=max_tokens, temperature=0.7, n=n_instruct)
-        self.eval_llm = OpenAI(openai_api_key=openai_key, max_tokens=max_tokens, temperature=0.7, echo=True)
+    def __init__(self, openai_key, n_instruct):
+        self.gen_llm = ChatOpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, n=n_instruct)
+        self.eval_llm = OpenAI(openai_api_key=openai_key, max_tokens=2000, temperature=0.7, echo=True)
         self.gen_chain = None
         self.eval_chain = None
         self.init()

ape/prompt.py

@@ -5,15 +5,13 @@ few_shot_prompt = "Input: {input}\nOutput: {output}"
 
 gen_user_prompt = '{few_shot}'
 
-gen_sys_prompt = """
-                I want you to act as an AI assisted doctor. You are capable of answering anything related to medical. Given 
-                a list of input-output pairs, you must come up with the correct instruction in medical-related area.
-                You must respond in the following format, and always respond in chinese.
-                ```
-                {{'instruction':"$YOUR_INSTRUCTION"}}
-                ```
-                Everything between the ``` must be valid json.
-                """
+gen_sys_prompt = """I want you to act as an AI assisted doctor. You are capable of answering anything related to medical. Given 
+a list of input-output pairs, you must come up with the correct instruction in medical-related area.
+You must respond in the following format, and always respond in chinese.
+```
+{{'instruction':"$YOUR_INSTRUCTION"}}
+```
+Everything between the ``` must be valid json. """
 
 eval_prompt = "Instruction: {prompt}\nInput: {input}\nOutput: {output}"
 

app.py

@@ -6,13 +6,13 @@ from ape.prompt import MyTemplate
 from ape.ape import *
 
 
-with gr.Blocks(title="Automatic Prompt Engineer", css=None) as demo:
+with gr.Blocks(title="Automatic Prompt Engineer", theme=gr.themes.Glass()) as demo:
     gr.Markdown("# Automatic Prompt Engineer")
-    openai_key = gr.Textbox(type='password', label='输入 API key')
-
     with gr.Row():
         with gr.Column(scale=2):
             gr.Markdown("## Configuration")
+            with gr.Row():
+                openai_key = gr.Textbox(type='password', label='输入 API key')
             with gr.Row():
                 n_train = gr.Slider(label="Number of Train", minimum=1, maximum=20, step=1, value=5)
                 n_few_shot = gr.Slider(label="Number of FewShot", minimum=1, maximum=20, step=1, value=5)
@@ -22,7 +22,7 @@ with gr.Blocks(title="Automatic Prompt Engineer", css=None) as demo:
                 n_instruct = gr.Slider(label="Number of Prompt", minimum=1, maximum=5, step=1, value=2)
 
         with gr.Column(scale=3):
-            gr.Markdown("## 加载数据集")
+            gr.Markdown("## Load Data")
             with gr.Tab("Choose Dataset"):
                 with gr.Row():
                     file = gr.File(label='上传txt文件，input\toutput\n', file_types=['txt'])
@@ -31,7 +31,9 @@ with gr.Blocks(title="Automatic Prompt Engineer", css=None) as demo:
                 with gr.Row():
                     instance = gr.State()
                     load_button = gr.Button("Load Task")
+                    load_flag = gr.Textbox()
                     sample_button = gr.Button('sample Data')
+                    sample_flag = gr.Textbox()
 
             with gr.Tab("Display Sampled Dataset"):
                 with gr.Row():
@@ -39,41 +41,31 @@ with gr.Blocks(title="Automatic Prompt Engineer", css=None) as demo:
                     eval_str = gr.Textbox(max_lines=100, lines=10, label="Data for scoring")
 
     with gr.Row():
-        gr.Markdown("## Run APE")
         with gr.Column(scale=2):
-            with gr.Row():
-                gr.Markdown('1. Generate Prompt')
-                gr.Markdown(MyTemplate['gen_sys_prompt'])
-                gr.Markdown('2. Evaluate Prompt')
-                gr.Markdown(MyTemplate['eval_prompt'])
+            gr.Markdown("## Run APE")
+            gen_prompt = gr.Textbox(max_lines=100, lines=10,
+                                    value=MyTemplate['gen_sys_prompt'], label="Prompt for generation")
+            eval_prompt = gr.Textbox(max_lines=100, lines=10,
+                                     value=MyTemplate['eval_prompt'], label="Prompt for Evaluation")
 
             with gr.Row():
-                basic_cost = gr.Textbox(lines=1, value="", label="Estimated Cost ($)", disabled=True)
-                basic_cost_button = gr.Button("Estimate Cost")
-                basic_ape_button = gr.Button("Run APE")
+                cost = gr.Textbox(lines=1, value="", label="Estimated Cost ($)")
+                cost_button = gr.Button("Estimate Cost")
+                ape_button = gr.Button("Run APE")
 
         with gr.Column(scale=3):
+            gr.Markdown("## Get Result")
             with gr.Tab("APE Results"):
+                all_prompt = gr.Textbox(label='Generated Prompt')
                 # Display all generated prompt with log probs
                 output_df = gr.DataFrame(type='pandas', headers=['Prompt', 'Likelihood'], wrap=True, interactive=False)
 
-            with gr.Tab("Prompt Overview"):
-                with gr.Row():
-                    generation_prompt_sample = gr.Textbox(lines=8, value="",
-                                                          label="Instruction Generation Prompts",
-                                                          disabled=True)
-                    evaluation_prompt_sample = gr.Textbox(lines=8, value="",
-                                                          label="Evaluation Prompts",
-                                                          disabled=True)
-
             with gr.Tab("Test Prompt"):
                 # Test the output of LLM using prompt
                 with gr.Row():
                     with gr.Column(scale=1):
-                        test_prompt = gr.Textbox(lines=4, value="",
-                                                 label="Prompt to test")
-                        test_inputs = gr.Textbox(lines=1, value="",
-                                                 label="Input used to test prompt")
+                        test_prompt = gr.Textbox(lines=4, value="", label="Prompt to test")
+                        test_inputs = gr.Textbox(lines=1, value="", label="Input used to test prompt")
                         answer_button = gr.Button("Test")
                     with gr.Column(scale=1):
                         test_output = gr.Textbox(lines=9, value="", label="Model Output")
@@ -93,16 +85,21 @@ with gr.Blocks(title="Automatic Prompt Engineer", css=None) as demo:
     Callback
     """
     # 1. 选择已有任务/上传文件，实例化Instance
-    load_button.click(load_task, [task, file], [instance])
+    load_button.click(load_task, [task, file], [instance, load_flag])
 
     # 2. 按 Configuration Sample数据 得到训练样本和验证集, 并在前端展示。支持重采样
-    sample_button.click(sample_data, [instance, n_train, n_few_shot, n_eval], [train_str, eval_str, instance])
+    sample_button.click(sample_data, [instance, n_train, n_few_shot, n_eval], [train_str, eval_str, instance, sample_flag])
 
     # 3. Estimate Cost for train + Eval
+    cost_button.click(esttimate_cost, [instance], [cost])
+
+    # 4. Run APE -> 所有指令
+    ape_button.click(generate, [instance, openai_key, n_instruct], [all_prompt])
 
+    # 5. Evaluate -> 得到所有指令的Log Prob
 
-    # 4. Run APE -> 所有指令，以及指令对应的log prob
+    # 6. 输入指令单测
 
-    # 5. 指令单测
+    # 7. 输入指令打分
 
-    # 6. 人工指令打分
+    demo.launch(show_error=True)