add Anthropic and OpenAI API

.gitignore

@@ -0,0 +1,17 @@
+# standard python ignores
+*.pyc
+__pycache__
+
+# virtualenv
+venv/
+.venv3/
+
+# IDE files
+.idea/
+.vscode/
+
+# macOS files
+.DS_Store
+
+# Environment
+.env

app.py

@@ -1,63 +1,111 @@
+import anthropic
+from openai import OpenAI
 import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
+import os
+from prompts import codebook
+
+_PROMPT_STYLES = [prompt['prompt_style'] for prompt in codebook]
+
+
+# Function to call Claude API
+def call_claude_api(input_text, prompt):
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    client = anthropic.Anthropic(api_key=api_key)
+
+    responses = []
+    messages = []
+    print("Using prompt style: ", prompt['prompt_style'])
+
+    for prompt_message in prompt['prompts']:
+        message = {"role": "user", "content": prompt_message}
+        messages.append(message)
+
+        response = client.messages.create(
+            model="claude-3-5-sonnet-20240620",
+            max_tokens=1024,
+            temperature=0,
+            system=prompt['system_prompt'] + input_text,
+            messages=messages
+        )
+
+        messages.append({
+            "role": "assistant",
+            "content": response.content[0].text
+        })
+        responses.append(response.content[0].text)
+
+    return responses
+
+
+def call_openai_api(input_text, prompt):
+    api_key = os.environ.get("OPENAI_API_KEY")
+    org_id = os.environ.get("OPENAI_ORG_ID")
+    project_id = os.environ.get("OPENAI_PROJECT_ID")
+
+    client = OpenAI(
+        api_key=api_key,
+        organization=org_id,
+        project=project_id
+    )
+
+    responses = []
+    messages = [{
+        "role": "system", "content": prompt['system_prompt'] + input_text
+    }]
+    print("Using prompt style: ", prompt['prompt_style'])
 
+    for prompt_message in prompt['prompts']:
+        message = {"role": "user", "content": prompt_message}
+        messages.append(message)
+
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            max_tokens=1024,
+            temperature=0,
+            messages=messages
+        )
+
+        messages.append({
+            "role": "assistant",
+            "content": response.choices[0].message.content
+        })
+        responses.append(response.choices[0].message.content)
+
+    return responses
+
+
+def process_file(text_file, prompt_style_key):
+    with open(text_file.name, "r") as fd:
+        content = fd.read()
+
+    for prompt in codebook:
+        if prompt['prompt_style'] == prompt_style_key:
+            selected_prompt = prompt
+            break
+
+    # responses = call_claude_api(content, prompt=selected_prompt)
+    responses = call_openai_api(content, prompt=selected_prompt)
+    sections = selected_prompt["sections"]
+
+    final_response = ''
+    for section, response in zip(sections, responses):
+        final_response += f"# {section}\n{response}\n\n"
+    return final_response
+
+
+# Gradio Components
+file_upload = gr.File(label="Upload your paper (only .txt files supported)")
+dropdown = gr.Dropdown(choices=list(_PROMPT_STYLES), label="Research domain", value=_PROMPT_STYLES[0])
+output = gr.Textbox(label="Critique")
+
+# Gradio Interface
+iface = gr.Interface(
+    fn=process_file,
+    inputs=[file_upload, dropdown],
+    outputs=output,
+    title="Alignment Research Critiquer",
+    description="An LLM-based app to critique AI alignment research papers."
+)
 
 if __name__ == "__main__":
-    demo.launch()
+    iface.launch()

prompts.py

@@ -0,0 +1,36 @@
+codebook = [
+    {
+        "prompt_style": "alignment (general)",
+        "system_prompt": "In this conversation, you will play the role of an experienced researcher critiquing research papers within the field of AI technical alignment. In your role, your priority is to be particularly critical and look out for holes in the authors’ arguments or empirical work for the purpose of helping early career AI safety researchers. Be direct, to the point, and concise. Here is the paper: ",
+        "prompts": [
+            "First, please provide an overview of the paper's claims. Extract text from the paper as evidence. Explain your reasoning for each step in your response.",
+            "Now, provide an overview of the paper's study design and methodology. Include an answer to the following question: Are the compute / data  requirements stated and comparable across methods and benchmarks? If not, is this discussed in the paper? Extract text from the paper as evidence. Explain your reasoning for each step in your response.",
+            "Provide an overview of the results, discussion, limitations, and future directions for research from the paper. Include an answer to the question “Are the claims following logically throughout the paper? Extract text from the paper as evidence. Explain your reasoning for each step in your response.",
+            "Next, critique the study design and methodology. Include a comprehensive answer to questions, “What evidence do the authors provide that their method performs well? “Does the empirical evidence support the claims? Provide an overview of the specific strengths of the paper. Extract text from the paper as evidence. Explain your reasoning for each step in your response.",
+            "Now, provide a comprehensive list of the weaknesses, how it weakens the evidence of the paper's claim, and why it does so. Include an answer to the following question in detail, “Is the method tested on a sufficiently broad set of benchmarks? “ Extract text from the paper as evidence. Explain your reasoning for each step in your response.",
+            "Critique the Results, Discussion, and limitations of the paper. Include an answer for the question 'Are the results statistically significant and does the evaluation protocol follow best practices (reporting confidence intervals, significance testing)?'",
+            "Now, please provide a comprehensive list of possible research directions for AI safety researchers. Include numerous, specific and narrowly scoped technical research questions to for exploratory analysis in each research direction. Choose research questions that will most benefit the field of AI Safety. Explain your reasoning for each step."
+        ],
+        "sections": [
+            "Paper Overview and Claims",
+            "Study Design and Methodology",
+            "Results and Discussion",
+            "Methodology Critique",
+            "Weaknesses and Evidence Impact",
+            "Results and Statistical Significance",
+            "Future Research Directions"
+        ]
+    },
+    {
+        "prompt_style": "mech_interp",
+        "system_prompt": "Here is the paper: ",
+        "prompts": [
+            "I've attached a paper on mechanistic interpretability of large language models. I'm submitting the paper soon and would like to practice responding to peer review, please give me a high quality response in the style of a thoughtful and careful reviewer of a prestigious conference like ICLR / ICML / NeurIPS about the strengths and weaknesses of the paper, and what can be improved about it.",
+            "Please also generate a low-effort and hostile response that misses the point of the paper, in the style of the proverbial 'reviewer 2'. I want practice preparing myself for the worst."
+        ],
+        "sections": [
+            "High Quality Review",
+            "Low Quality Review"
+        ]
+    }
+]