Initial import

.gitignore

@@ -0,0 +1 @@
+config.py

README.md

@@ -10,4 +10,4 @@ pinned: false
 license: bigscience-bloom-rail-1.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+A basic example of dynamic adversarial data collection with a Gradio app.

app.py

@@ -0,0 +1,82 @@
+# Basic example for doing model-in-the-loop dynamic adversarial data collection
+# using Gradio Blocks.
+
+import random
+from urllib.parse import parse_qs
+
+import gradio as gr
+import requests
+from transformers import pipeline
+
+demo = gr.Blocks()
+
+with demo:
+    total_cnt = 2 # How many examples per HIT
+    dummy = gr.Textbox(visible=False)  # dummy for passing assignmentId
+
+    # We keep track of state as a Variable
+    state_dict = {"assignmentId": "", "cnt": 0, "fooled": 0, "data": [], "metadata": {}}
+    state = gr.Variable(state_dict)
+
+    gr.Markdown("# DADC in Gradio example")
+    gr.Markdown("Try to fool the model and find an example where it predicts the wrong label!")
+
+    state_display = gr.Markdown(f"State: 0/{total_cnt} (0 fooled)")
+
+    # Generate model prediction
+    # Default model: distilbert-base-uncased-finetuned-sst-2-english
+    def _predict(txt, tgt, state):
+        pipe = pipeline("sentiment-analysis")
+        pred = pipe(txt)[0]
+
+        pred["label"] = pred["label"].title()
+        ret = f"Target: {tgt}. Model prediction: {pred['label']} ({pred['score']} confidence). {pred['label'] != tgt}\n\n"
+        if pred["label"] != tgt:
+            state["fooled"] += 1
+            ret += " You fooled the model! Well done!"
+        else:
+            ret += " You did not fool the model! Too bad, try again!"
+        state["data"].append(ret)
+        state["cnt"] += 1
+
+        done = state["cnt"] == total_cnt
+        toggle_final_submit = gr.update(visible=done)
+        toggle_example_submit = gr.update(visible=not done)
+        new_state_md = f"State: {state['cnt']}/{total_cnt} ({state['fooled']} fooled)"
+        return ret, state, toggle_example_submit, toggle_final_submit, new_state_md
+
+    # Input fields
+    text_input = gr.Textbox(placeholder="Enter model-fooling statement", show_label=False)
+    labels = ["Positive", "Negative"]
+    random.shuffle(labels)
+    label_input = gr.Radio(choices=labels, label="Target (correct) label")
+    text_output = gr.Markdown()
+    with gr.Column() as example_submit:
+        submit_ex_button = gr.Button("Submit")
+    with gr.Column(visible=False) as final_submit:
+        submit_hit_button = gr.Button("Submit HIT")
+
+    # Submit state to MTurk backend for ExternalQuestion
+    # Update the URL below to switch from Sandbox to real data collection
+    def _submit(state, dummy):
+        query = parse_qs(dummy[1:])
+        assert "assignmentId" in query, "No assignment ID provided, unable to submit"
+        state["assignmentId"] = query["assignmentId"]
+        url = "https://workersandbox.mturk.com/mturk/externalSubmit"
+        return requests.post(url, data=state)
+
+    # Button event handlers
+    submit_ex_button.click(
+        _predict,
+        inputs=[text_input, label_input, state],
+        outputs=[text_output, state, example_submit, final_submit, state_display],
+    )
+
+    submit_hit_button.click(
+        _submit,
+        inputs=[state, dummy],
+        outputs=None,
+        _js="function(state, dummy) { return [state, window.location.search]; }",
+    )
+
+demo.launch()

collect.py

@@ -0,0 +1,39 @@
+# Basic example for running MTurk data collection against a Space
+# For more information see https://docs.aws.amazon.com/mturk/index.html
+
+import boto3
+from boto.mturk.question import ExternalQuestion
+
+from config import MTURK_KEY, MTURK_SECRET
+
+MTURK_REGION = "us-east-1"
+MTURK_SANDBOX = "https://mturk-requester-sandbox.us-east-1.amazonaws.com"
+
+mturk = boto3.client(
+    "mturk",
+    aws_access_key_id=MTURK_KEY,
+    aws_secret_access_key=MTURK_SECRET,
+    region_name=MTURK_REGION,
+    endpoint_url=MTURK_SANDBOX,
+)
+
+question = ExternalQuestion(
+    "https://huggingface.co/spaces/douwekiela/dadc", frame_height=600
+)
+
+new_hit = mturk.create_hit(
+    Title="DADC with Gradio",
+    Description="Hello world",
+    Keywords="fool the model",
+    Reward="0.15",
+    MaxAssignments=1,
+    LifetimeInSeconds=172800,
+    AssignmentDurationInSeconds=600,
+    AutoApprovalDelayInSeconds=14400,
+    Question=question.get_as_xml(),
+)
+
+print(
+    "Sandbox link: https://workersandbox.mturk.com/mturk/preview?groupId="
+    + new_hit["HIT"]["HITGroupId"]
+)

config.py.example

@@ -0,0 +1,6 @@
+# Fill in the information and rename this file config.py
+# You can obtain the key and secret in the AWS Identity
+# and Access Management (IAM) panel.
+
+MTURK_KEY = ''
+MTURK_SECRET = ''

requirements.txt

@@ -0,0 +1,2 @@
+requests
+transformers