Add evolutionary method support

app.py

@@ -7,12 +7,11 @@ import zipfile
 from io import StringIO
 
 import pandas as pd
-import requests
 import streamlit as st
 from datasets import load_dataset
 from gretel_client import Gretel
 from navigator_helpers import (
-    DataSynthesisConfig,
+    InstructionResponseConfig,
     TrainingDataSynthesizer,
     StreamlitLogHandler,
 )
@@ -31,50 +30,40 @@ logger.addHandler(handler)
 
 
 SAMPLE_DATASET_URL = "https://gretel-public-website.s3.us-west-2.amazonaws.com/datasets/llm-training-data/dolly-examples-qa-with-context.csv"
-WELCOME_MARKDOWN = """
-Gretel Navigator is a compound AI system designed to help you create high-quality, diverse training data examples through synthetic data generation techniques. It aims to assist in scenarios where you have limited training data or want to enhance the quality and diversity of your existing dataset.
+WELCOME_MARKDOWN = """Gretel Navigator is an advanced AI system for generating high-quality, diverse synthetic data to train AI models and LLMs. It combines cutting-edge techniques from recent research with Gretel's proprietary methods to enhance your training data.
 
-Key Use Cases
+### 🌟 Key Features & Techniques
 
-1. **Create Diverse Training or Evaluation Data from a seed**: Generate diverse training or evaluation data from plain text or seed examples. This ensures your AI models are exposed to a wide range of scenarios and edge cases during training.
-2. **Enhance Limited Training Data**: Expand your existing training data with additional synthetic examples generated by Gretel Navigator. This can help improve the robustness and generalization of your AI models.
-3. **Mitigate Bias and Toxicity**: Generate training examples that are unbiased and non-toxic by incorporating diverse perspectives and adhering to ethical guidelines. This promotes fairness and responsible AI development.
-4. **Enhance Model Performance**: Improve the performance of your AI models across various tasks by training them on domain specific synthetic data generated by Gretel Navigator.
+- **Evolutionary Text Generation**: Inspired by WizardLM-2's diverse knowledge generation
+- **AI-Aligning-AI (AAA)**: Leveraging concepts from Self-Rewarding Language Models
+- **Quality Evaluation & Ranking**: Using Gretel's proprietary scoring methods
+- **Instruction-Response Generation**: Influenced by StarCoder2-Instruct's approach
+- **Comprehensive Training Data**: Inspired by "Textbooks Are All You Need II"
 
-## 🌟 Synthetic Data Generation 
+### 🚀 Use Cases
 
-Gretel Navigator utilizes an agent-based system to generate high-quality synthetic data:
+1. Create diverse training/evaluation data from seeds
+2. Enhance limited datasets
+3. Mitigate bias and toxicity
+4. Improve model performance with domain-specific data
 
-- Diverse Instruction and Response Generation
-- Quality Evaluation and Ranking
-- AI-Aligning-AI Methodology (AAA) for iterative data quality enhancement
-- Co-teach, suggestions, and self-teaching for iterative improvement.
+### 🔧 How It Works
 
-Leveraging these techniques, Gretel Navigator helps you create training data that leads to more robust, unbiased, and high-performing AI models.
+1. Initialize with custom configuration
+2. Generate and evolve text populations
+3. Apply AI Align AI (AAA) for quality enhancement
+4. Evaluate and output high-quality synthetic data
 
-## 🔧 Getting Started
+### 📂 Input & Output
 
-To start using Gretel Navigator, you'll need:
+- **Input**: Seed data (text or input/output pairs) in various formats (CSV, JSON, JSONL, Hugging Face datasets)
+- **Output**: High-quality synthetic training examples
 
-1. A Gretel account (free accounts are available).
-2. Seed text or input/output pairs to create or augment AI training data.
-
-## 📂 Input Data Formats
-
-Gretel Navigator supports the following formats for input data:
-
-- Seed data
-  - Input/Output pairs (or instruction/response) with any number of ground truth or "context fields".
-  - Plain text (ground truth data)
-- File formats: Hugging Face dataset, CSV, JSON, JSONL
-
-## 📤 Output
-
-Gretel Navigator generates one additional training example per row in the input/output pair format. You can specify requirements for the input and output pairs in the configuration. Run the process multiple times to scale your data to any desired level.
+Ready to elevate your AI training data? Let's get started with Gretel Navigator! 🚀
 
 ---
 
-Ready to enhance your AI training data and unlock the full potential of your models? Let's get started with Gretel Navigator! 🚀
+*Gretel Navigator combines techniques from recent academic research with Gretel's innovative approaches to deliver state-of-the-art synthetic data generation.*
 """
 
 
@@ -210,17 +199,20 @@ def main():
             st.session_state.selected_fields = selected_fields
 
         with st.expander("Advanced Options", expanded=False):
-
             output_instruction_field = st.text_input(
                 "Synthetic instruction field",
-                value=st.session_state.get("output_instruction_field", "instruction"),
+                value=st.session_state.get(
+                    "output_instruction_field", "synthetic_instruction"
+                ),
                 help="Specify the name of the output field for generated instructions",
             )
             st.session_state.output_instruction_field = output_instruction_field
 
             output_response_field = st.text_input(
                 "Synthetic response field",
-                value=st.session_state.get("output_response_field", "response"),
+                value=st.session_state.get(
+                    "output_response_field", "synthetic_response"
+                ),
                 help="Specify the name of the output field for generated responses",
             )
             st.session_state.output_response_field = output_response_field
@@ -234,76 +226,67 @@ def main():
             )
             st.session_state.num_records = num_records
 
-            num_instructions = st.number_input(
-                "Number of diverse candidate instructions",
+            num_generations = st.number_input(
+                "Number of generations",
                 min_value=1,
-                value=st.session_state.get("num_instructions", 5),
-                help="Specify the number of instructions to generate",
+                value=st.session_state.get("num_generations", 3),
+                help="Specify the number of generations for the evolutionary algorithm",
             )
-            st.session_state.num_instructions = num_instructions
+            st.session_state.num_generations = num_generations
 
-            num_responses = st.number_input(
-                "Number of diverse candidateresponses",
+            population_size = st.number_input(
+                "Population size",
                 min_value=1,
-                value=st.session_state.get("num_responses", 5),
-                help="Specify the number of responses to generate",
+                value=st.session_state.get("population_size", 5),
+                help="Specify the population size for the evolutionary algorithm",
+            )
+            st.session_state.population_size = population_size
+
+            mutation_rate = st.slider(
+                "Mutation rate",
+                min_value=0.0,
+                max_value=1.0,
+                value=st.session_state.get("mutation_rate", 0.5),
+                step=0.1,
+                help="Adjust the mutation rate for the evolutionary algorithm",
             )
-            st.session_state.num_responses = num_responses
+            st.session_state.mutation_rate = mutation_rate
 
             temperature = st.slider(
                 "Temperature",
                 min_value=0.0,
                 max_value=1.0,
-                value=st.session_state.get("temperature", 0.8),
+                value=st.session_state.get("temperature", 0.7),
                 step=0.1,
                 help="Adjust the temperature for response generation",
             )
             st.session_state.temperature = temperature
 
-            max_tokens_instruction = st.slider(
-                "Max tokens (instruction)",
+            max_tokens = st.slider(
+                "Max tokens",
                 min_value=1,
                 max_value=1024,
-                value=st.session_state.get("max_tokens_instruction", 100),
-                help="Specify the maximum number of tokens for instructions",
+                value=st.session_state.get("max_tokens", 200),
+                help="Specify the maximum number of tokens for generated text",
             )
-            st.session_state.max_tokens_instruction = max_tokens_instruction
-
-            max_tokens_response = st.slider(
-                "Max tokens (response)",
-                min_value=1,
-                max_value=1024,
-                value=st.session_state.get("max_tokens_response", 100),
-                help="Specify the maximum number of tokens for responses",
-            )
-            st.session_state.max_tokens_response = max_tokens_response
+            st.session_state.max_tokens = max_tokens
 
         with st.expander("Model Configuration", expanded=True):
             st.markdown("### Primary Navigator Models")
 
-            tabular_models = st.session_state.gretel.factories.get_navigator_model_list(
-                "tabular"
-            )
             navigator_tabular = st.selectbox(
                 "Navigator Tabular",
-                options=tabular_models,
-                index=st.session_state.get("navigator_tabular_index", 0),
+                options=["gretelai/auto"],
+                index=0,
                 help="Select the primary Navigator tabular model",
             )
-            st.session_state.navigator_tabular_index = tabular_models.index(
-                navigator_tabular
-            )
 
-            nl_models = st.session_state.gretel.factories.get_navigator_model_list(
-                "natural_language"
-            )
             navigator_llm = st.selectbox(
                 "Navigator LLM",
-                options=nl_models,
-                index=st.session_state.get("navigator_llm_index", 0),
+                options=["gretelai/gpt-auto", "gretelai/gpt-llama3-8b"],
+                index=0,
                 help="Select the primary Navigator LLM",
             )
-            st.session_state.navigator_llm_index = nl_models.index(navigator_llm)
 
             st.markdown("---")
             st.markdown("### AI Align AI (AAA)")
@@ -318,7 +301,7 @@ def main():
             )
             st.session_state.use_aaa = use_aaa
 
-            co_teach_llms = []  # Initialize co_teach_llms with an empty list
+            co_teach_llms = []
 
             if use_aaa:
                 st.markdown("#### Navigator Co-teaching LLMs")
@@ -326,13 +309,10 @@ def main():
                     "Select additional Navigator LLMs for co-teaching in AAA. It is recommended to use different LLMs than the primary Navigator LLM for this step."
                 )
 
-                for model in nl_models:
-                    if model != navigator_llm:
-                        if st.checkbox(model, value=True, key=f"checkbox_{model}"):
-                            co_teach_llms.append(model)
-                    else:
-                        if st.checkbox(model, value=False, key=f"checkbox_{model}"):
-                            co_teach_llms.append(model)
+                co_teach_options = ["gretelai/gpt-llama3-8b", "gretelai/gpt-mistral7b"]
+                for model in co_teach_options:
+                    if st.checkbox(model, value=True, key=f"checkbox_{model}"):
+                        co_teach_llms.append(model)
                 st.session_state.co_teach_llms = co_teach_llms
 
             st.markdown("---")
@@ -342,10 +322,7 @@ def main():
                 "System Prompt",
                 value=st.session_state.get(
                     "system_prompt",
-                    "You are an AI assistant tasked with generating high-quality instruction-response pairs.\n"
-                    "Your goal is to create diverse, engaging, and informative content that covers a wide range of topics.\n"
-                    "When generating instructions, aim for clear, concise questions or commands that prompt thoughtful responses.\n"
-                    "When generating responses, provide detailed, accurate, and helpful information that directly addresses the instruction.",
+                    "You are an expert in generating balanced, context-rich questions and comprehensive answers based on given contexts. Your goal is to create question-answer pairs that are informative, detailed when necessary, and understandable without prior knowledge, while not revealing the answer in the question.",
                 ),
                 help="Specify the system prompt for the LLM",
             )
@@ -355,32 +332,95 @@ def main():
                 "Instruction Format Prompt",
                 value=st.session_state.get(
                     "instruction_format_prompt",
-                    "A well-formulated question or command in everyday English.",
+                    "Generate a specific and clear question directly related to a key point in the given context. The question should include enough background information to be understood without prior knowledge, while being answerable using only the information provided. Do not reveal the answer in the question. Ensure the question is focused and can be answered concisely if the information allows, but also accommodate for more detailed responses when appropriate.",
                 ),
                 help="Specify the format prompt for instructions",
             )
             st.session_state.instruction_format_prompt = instruction_format_prompt
 
+            instruction_mutation_prompt = st.text_area(
+                "Instruction Mutation Prompt",
+                value=st.session_state.get(
+                    "instruction_mutation_prompt",
+                    "Refine this question to include necessary context for understanding, without revealing the answer. Ensure it remains clear and can be comprehensively answered using only the information in the given context. Adjust the question to allow for a concise answer if possible, but also consider if a more detailed response is warranted based on the complexity of the topic.",
+                ),
+                help="Specify the mutation prompt for instructions",
+            )
+            st.session_state.instruction_mutation_prompt = instruction_mutation_prompt
+
+            instruction_quality_prompt = st.text_area(
+                "Instruction Quality Prompt",
+                value=st.session_state.get(
+                    "instruction_quality_prompt",
+                    "Evaluate the quality of this question based on its specificity, inclusion of necessary context, relevance to the original context, clarity for someone unfamiliar with the topic, and ability to be answered appropriately (either concisely or in detail) without revealing the answer:",
+                ),
+                help="Specify the quality evaluation prompt for instructions",
+            )
+            st.session_state.instruction_quality_prompt = instruction_quality_prompt
+
+            instruction_complexity_target = st.slider(
+                "Instruction Complexity Target",
+                min_value=0.0,
+                max_value=1.0,
+                value=st.session_state.get("instruction_complexity_target", 0.7),
+                step=0.1,
+                help="Specify the target complexity for instructions",
+            )
+            st.session_state.instruction_complexity_target = (
+                instruction_complexity_target
+            )
+
             response_format_prompt = st.text_area(
                 "Response Format Prompt",
                 value=st.session_state.get(
                     "response_format_prompt",
-                    "A well-formulated response to the question in everyday English.",
+                    "Generate an informative answer to the given question. Use only the information provided in the original context. The response should be as concise as possible while fully addressing the question, including relevant context and explanations where necessary. For complex topics, provide a more detailed response. Ensure the answer provides enough background information to be understood by someone unfamiliar with the topic.",
                 ),
                 help="Specify the format prompt for responses",
             )
             st.session_state.response_format_prompt = response_format_prompt
 
+            response_mutation_prompt = st.text_area(
+                "Response Mutation Prompt",
+                value=st.session_state.get(
+                    "response_mutation_prompt",
+                    "Refine this answer to balance conciseness with comprehensiveness. For straightforward questions, aim for brevity while ensuring accuracy. For complex topics, provide more detail and context. Add relevant information from the context as needed. Verify factual accuracy and correct any inaccuracies or missing key information. Ensure the answer can be understood without prior knowledge of the topic.",
+                ),
+                help="Specify the mutation prompt for responses",
+            )
+            st.session_state.response_mutation_prompt = response_mutation_prompt
+
+            response_quality_prompt = st.text_area(
+                "Response Quality Prompt",
+                value=st.session_state.get(
+                    "response_quality_prompt",
+                    "Evaluate the quality of this answer based on its accuracy, appropriate level of detail (concise for simple questions, comprehensive for complex ones), relevance to the question, clarity for someone unfamiliar with the topic, inclusion of necessary background information, and whether it provides a satisfactory response using only the information from the given context:",
+                ),
+                help="Specify the quality evaluation prompt for responses",
+            )
+            st.session_state.response_quality_prompt = response_quality_prompt
+
+            response_complexity_target = st.slider(
+                "Response Complexity Target",
+                min_value=0.0,
+                max_value=1.0,
+                value=st.session_state.get("response_complexity_target", 0.8),
+                step=0.1,
+                help="Specify the target complexity for responses",
+            )
+            st.session_state.response_complexity_target = response_complexity_target
+
         with st.expander("Download SDK Code", expanded=False):
             st.markdown("### Ready to generate data at scale?")
             st.write(
                 "Get started with your current configuration using the SDK code below:"
             )
+
             config_text = f"""#!pip install -Uqq git+https://github.com/gretelai/navigator-helpers.git
 
 import logging
 import pandas as pd
-from navigator_helpers import DataSynthesisConfig, TrainingDataSynthesizer
+from navigator_helpers import InstructionResponseConfig, TrainingDataSynthesizer
 
 # Configure the logger
 logging.basicConfig(level=logging.INFO, format="%(message)s")
@@ -390,34 +430,42 @@ API_KEY = "YOUR_API_KEY"
 
 df = pd.read_csv(DATASET)
 
-# Create the data synthesis configuration
-config = DataSynthesisConfig(
+# Create the instruction response configuration
+config = InstructionResponseConfig(
     input_fields={st.session_state.selected_fields},
     output_instruction_field="{output_instruction_field}",
     output_response_field="{output_response_field}",
-    num_instructions={num_instructions},
-    num_responses={num_responses},
+    num_generations={num_generations},
+    population_size={population_size},
+    mutation_rate={mutation_rate},
     temperature={temperature},
-    max_tokens_instruction={max_tokens_instruction},
-    max_tokens_response={max_tokens_response},
+    max_tokens={max_tokens},
     api_key=API_KEY,
     navigator_tabular="{navigator_tabular}",
     navigator_llm="{navigator_llm}",
     co_teach_llms={co_teach_llms},
     system_prompt='''{system_prompt}''',
     instruction_format_prompt='''{instruction_format_prompt}''',
-    response_format_prompt='''{response_format_prompt}'''
+    instruction_mutation_prompt='''{instruction_mutation_prompt}''',
+    instruction_quality_prompt='''{instruction_quality_prompt}''',
+    instruction_complexity_target={instruction_complexity_target},
+    response_format_prompt='''{response_format_prompt}''',
+    response_mutation_prompt='''{response_mutation_prompt}''',
+    response_quality_prompt='''{response_quality_prompt}''',
+    response_complexity_target={response_complexity_target},
+    use_aaa={use_aaa}
 )
 
 # Create the training data synthesizer and perform synthesis
 synthesizer = TrainingDataSynthesizer(
     df,
     config,
-    use_aaa={use_aaa},
-    output_file="results.csv",
+    output_file="results.jsonl",
     verbose=True,
 )
-new_df = synthesizer.generate()"""
+new_df = synthesizer.generate()
+"""
+
             st.code(config_text, language="python")
             st.download_button(
                 label="Download SDK Code",
@@ -473,22 +521,29 @@ new_df = synthesizer.generate()"""
                 handler = StreamlitLogHandler(custom_log_handler)
                 logger.addHandler(handler)
 
-                config = DataSynthesisConfig(
+                config = InstructionResponseConfig(
                     input_fields=selected_fields,
                     output_instruction_field=output_instruction_field,
                     output_response_field=output_response_field,
-                    num_instructions=num_instructions,
-                    num_responses=num_responses,
+                    num_generations=num_generations,
+                    population_size=population_size,
+                    mutation_rate=mutation_rate,
                     temperature=temperature,
-                    max_tokens_instruction=max_tokens_instruction,
-                    max_tokens_response=max_tokens_response,
+                    max_tokens=max_tokens,
                     api_key=api_key,
                     navigator_tabular=navigator_tabular,
                     navigator_llm=navigator_llm,
                     co_teach_llms=co_teach_llms,
                     system_prompt=system_prompt,
                     instruction_format_prompt=instruction_format_prompt,
+                    instruction_mutation_prompt=instruction_mutation_prompt,
+                    instruction_quality_prompt=instruction_quality_prompt,
+                    instruction_complexity_target=instruction_complexity_target,
                     response_format_prompt=response_format_prompt,
+                    response_mutation_prompt=response_mutation_prompt,
+                    response_quality_prompt=response_quality_prompt,
+                    response_complexity_target=response_complexity_target,
+                    use_aaa=use_aaa,
                 )
 
                 start_time = time.time()
@@ -498,7 +553,6 @@ new_df = synthesizer.generate()"""
                         synthesizer = TrainingDataSynthesizer(
                             pd.DataFrame([row]),
                             config,
-                            use_aaa=use_aaa,
                             output_file="results.csv",
                             verbose=True,
                         )