Merge remote-tracking branch 'upstream/main'

README.md

@@ -15,7 +15,7 @@ This repo shares our implementation of **IndieLabel**—an interactive web appli
     ```
     $ pip install -r requirements.txt
     ```
-- Download and unzip the `data` sub-directory from [this Drive folder](https://drive.google.com/file/d/1js0HAwBpEcoRdT9nkGY0Mc5Falkpsonl/view?usp=sharing) and place it in the repo directory (39.7MB zipped, 124.6MB unzipped).
+- Download and unzip the `data` sub-directory from [this Drive folder](https://drive.google.com/file/d/1iYueqzG9qIB45HT_5iwJp-44Dhfm2XbR/view?usp=sharing) and place it in the repo directory (39.7MB zipped, 124.3MB unzipped).
 
 
 - Start the Flask server:

audit_utils.py

@@ -336,6 +336,10 @@ def format_description(indie_label_json):
     text_entry = indie_label_json["text_entry"]
     return f"Title: {title}\nError Type: {error_type}\nSummary/Suggestions: {text_entry}"
 
+# Format the credit field for the report with the current user's username, optional name, and optional email address
+def format_credit(cur_user, name, email):
+    return f"Username: {cur_user}, Name: {name}, Email: {email}"
+
 # Convert indielabel json to AVID json format.
 # See the AVID format in https://avidml.org/avidtools/reference/report
 #
@@ -349,7 +353,7 @@ def format_description(indie_label_json):
 #   user_rating                 personal_model_score    0.92
 #   user_decision               user_decision           "Non-toxic"
 # Note that this is at the individual report level.
-def convert_indie_label_json_to_avid_json(indie_label_json, cur_user, email, sep_selection):
+def convert_indie_label_json_to_avid_json(indie_label_json, cur_user, name, email, sep_selection):
 
     # Setting up the structure with a dict to enable programmatic additions
     avid_json_dict = { 
@@ -398,17 +402,16 @@ def convert_indie_label_json_to_avid_json(indie_label_json, cur_user, email, sep
               "taxonomy_version": "0.2"
             }
           },
-          "credit": "", # Leaving empty so that credit can be assigned
+          "credit": {
+            "lang": "eng", # TODO: Make language selectable
+            "value": "" # Leaving empty so that credit can be assigned
+          },
           "reported_date": "" # Leaving empty so that it can be dynamically filled in
     }
 
-    avid_json_dict["description"] = format_description(indie_label_json)
-    avid_json_dict["reported_date"] = str(date.today())
-    # Assign credit to email if provided, otherwise default to randomly assigned username
-    if email != "":
-        avid_json_dict["credit"] = email
-    else:
-        avid_json_dict["credit"] = cur_user
+    avid_json_dict["description"]["value"] = format_description(indie_label_json)
+    avid_json_dict["reported_date"] = str(date.today())    
+    avid_json_dict["credit"]["value"] = format_credit(cur_user, name, email)
 
     sep_enum = get_sep_enum(sep_selection)
     avid_json_dict["impact"]["avid"]["sep_view"] = [sep_enum]

indie_label_svelte/package.json

@@ -1,10 +1,10 @@
 {
-  "name": "svelte-vega-sample",
+  "name": "indie-label",
   "version": "0.0.0",
-  "author": "Alex Bäuerle <bauerlealex@gmail.com> (https://a13x.io)",
-  "repository": "git@github.com:vega/svelte-vega.git",
+  "author": "Michelle Lam <michelle123lam@gmail.com> (http://michelle123lam.github.io)",
+  "repository": "git@github.com:StanfordHCI/indie-label.git",
   "bugs": {
-    "url": "https://github.com/vega/svelte-vega/issues"
+    "url": "https://github.com/StanfordHCI/indie-label/issues"
   },
   "private": true,
   "scripts": {
@@ -25,7 +25,7 @@
     "@smui/checkbox": "^6.0.0-beta.2",
     "@smui/chips": "^6.0.0-beta.2",
     "@smui/circular-progress": "^6.0.0-beta.4",
-    "@smui/common": "^6.0.0-beta.2",
+    "@smui/common": "^6.1.4",
     "@smui/data-table": "^6.0.0-beta.2",
     "@smui/drawer": "^6.0.0-beta.4",
     "@smui/form-field": "^6.0.0-beta.2",

indie_label_svelte/public/global.css

@@ -120,7 +120,7 @@ h6 {
 	font-weight: bold;
 	font-size: 16px;
 	margin-top: 25px;
-	margin-bottom: 25px;
+	margin-bottom: 15px;
 }
 
 .head_6_non_cap {
@@ -204,6 +204,11 @@ table {
 	margin: 60px 0;
 }
 
+.spacing_vert_100_bottom {
+	margin: 0 0 100px 0;
+}
+
+
 .page_title {
 	font-size: 30px;
 	font-weight: bold;
@@ -319,4 +324,21 @@ table {
 
 .mdc-drawer .mdc-deprecated-list-item {
 	height: 64px !important;
-}
+}
+
+/* Material UI secondary button formatting */
+:root {
+	--mdc-theme-secondary: #676778;
+}
+.smui-button--color-secondary:not(:disabled) {
+    color: var(--mdc-theme-secondary) !important;
+}
+.smui-button--color-secondary.mdc-button--outlined:not(:disabled) {
+    border-color: var(--mdc-theme-secondary) !important;
+}
+.smui-button--color-secondary .mdc-button__ripple::before, .smui-button--color-secondary .mdc-button__ripple::after {
+	background-color: var(--mdc-theme-secondary) !important;
+}
+.smui-button--color-secondary:not(:disabled) .mdc-button__icon {
+	color: var(--mdc-theme-secondary) !important;
+}

indie_label_svelte/public/index.html

@@ -10,9 +10,6 @@
 	<link rel='icon' type='image/png' href='/favicon.png'>
 	<link rel='stylesheet' href='/global.css'>
 	<link rel='stylesheet' href='/build/bundle.css'>
-	<link rel='stylesheet' href='/build/extra.css'>
-	<!-- <link rel="stylesheet" href="../node_modules/svelte-material-ui/bare.css" /> -->
-	<!-- <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/svelte-material-ui@5.0.0/bare.min.css" /> -->
 	<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons" />
 
 	<link
@@ -22,6 +19,15 @@
 
 	<script defer src='/build/bundle.js'></script>
 </head>
+<!-- Google tag (gtag.js) -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-GFB1D1ZXRZ"></script>
+<script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+
+  gtag('config', 'G-GFB1D1ZXRZ');
+</script>
 
 <body>
 </body>

indie_label_svelte/rollup.config.js

@@ -51,7 +51,6 @@ export default {
     // we'll extract any component CSS out into
     // a separate file - better for performance
     css({ output: "bundle.css" }),
-    // css({ output: 'public/build/extra.css' }),
 
     // If you have external dependencies installed from
     // npm, you'll most likely need these plugins. In

indie_label_svelte/src/About.svelte

@@ -0,0 +1,225 @@
+<script lang="ts">
+</script>
+
+<svelte:head>
+    <title>About</title>
+</svelte:head>
+
+<div class="panel">
+    <div class="panel_contents">
+        <div>
+            <h3>IndieLabel Tutorial</h3>
+            <p class="body">
+                Check out this 5-minute video tutorial for a quick overview of the IndieLabel tool. If you prefer a written document, you can also find a help article <a href="https://docs.google.com/document/d/1BTo8LFUriiZcSWCcQy0lel9VxGl4KZtlFnu7wAFaiu4/edit?usp=sharing" target="_blank">here</a>.
+            </p>
+            <div class="video-container">
+                <iframe src="https://www.youtube.com/embed/Je0DCDnJ6KQ?si=H1dVy-oe6PSP0QYM" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+            </div>
+        </div>
+        <div>
+            <h3>About IndieLabel</h3>
+            <p class="body">
+                The IndieLabel deployment on this Hugging Face Space is a collaboration between researchers at the <a href="https://hci.stanford.edu/" target="_blank">Stanford HCI Group</a> and <a href="https://avidml.org/arva/" target="_blank">ARVA (AI Risk and Vulnerability Alliance)</a>. The IndieLabel tool is a research prototype designed to empower everyday users to lead large-scale investigations of harmful algorithmic behavior, and this Space is intended as a public-facing deployment of this prototype.
+            </p>
+
+            <div class="card-container">
+                <p class="head_5">IndieLabel & the End-User Audits paper</p>
+                <p class="body">
+                    The <a href="https://github.com/StanfordHCI/indie-label" target="_blank">IndieLabel</a> tool was initially presented as a research prototype in an academic publication called <b><i>End-User Audits: A System Empowering Communities to Lead Large-Scale Investigations of Harmful Algorithmic Behavior</i></b>. IndieLabel aids users in rapidly identifying where they disagree with a system's behavior with the help of a personalized recommender system model.
+                </p>
+                <p class="body">
+                    The key question underlying the End-User Audits paper is: <i>how do we enable everyday users to lead large-scale algorithm audits</i>? The full paper was presented at CSCW 2022 and is available on the <a href="https://dl.acm.org/doi/10.1145/3555625" target="_blank">ACM Digital Library</a> or via <a href="https://hci.stanford.edu/publications/2022/Lam_EndUserAudits_CSCW22.pdf" target="_blank">Stanford HCI</a>. The End-User Audits paper abstract summarizes our core motivation, approach, and findings:
+                </p>
+                <p class="body blockquote">
+                    "Because algorithm audits are conducted by technical experts, audits are necessarily limited to the hypotheses that experts think to test. End users hold the promise to expand this purview, as they inhabit spaces and witness algorithmic impacts that auditors do not. In pursuit of this goal, we propose end-user audits—system-scale audits led by non-technical users—and present an approach that scaffolds end users in hypothesis generation, evidence identification, and results communication. Today, performing a system-scale audit requires substantial user effort to label thousands of system outputs, so we introduce a collaborative filtering technique that leverages the algorithmic system's own disaggregated training data to project from a small number of end user labels onto the full test set. Our end-user auditing tool, IndieLabel, employs these predicted labels so that users can rapidly explore where their opinions diverge from the algorithmic system's outputs. By highlighting topic areas where the system is under-performing for the user and surfacing sets of likely error cases, the tool guides the user in authoring an audit report. In an evaluation of end-user audits on a popular comment toxicity model with 17 non-technical participants, participants both replicated issues that formal audits had previously identified and also raised previously underreported issues such as under-flagging on veiled forms of hate that perpetuate stigma and over-flagging of slurs that have been reclaimed by marginalized communities."
+                </p>
+                
+
+                <p class="body">
+                    The End-User Audits work was led by Stanford PhD student Michelle Lam along with co-authors Mitchell Gordon, Danaë Metaxa, Jeffrey Hancock, James Landay, and Michael Bernstein.
+                </p>
+
+                <div class="image-container">
+                    <div>
+                        <img src="./img/lam.jpg" alt="Michelle Lam">
+                        <p>
+                            <a href="http://michelle123lam.github.io" target="_blank"><b>Michelle Lam</b></a><br>
+                            PhD Student, Stanford CS
+                        </p>
+                    </div>
+                    <div>
+                        <img src="./img/gordon.jpeg" alt="Mitchell Gordon">
+                        <p>
+                            <a href="https://mgordon.me/" target="_blank"><b>Mitchell Gordon</b></a><br/>
+                            Incoming Asst Professor, MIT EECS
+                        </p>
+                    </div>
+                    <div>
+                        <img src="./img/metaxa.jpg" alt="Danaë Metaxa">
+                        <p>
+                            <a href="https://metaxa.net/" target="_blank"><b>Danaë Metaxa</b></a><br/>
+                            Asst Professor, UPenn CIS
+                        </p>
+                    </div>
+                    <div>
+                        <img src="./img/hancock.png" alt="Jeffrey Hancock">
+                        <p>
+                            <a href="https://sml.stanford.edu/people/jeff-hancock" target="_blank"><b>Jeffrey Hancock</b></a><br/>
+                            Professor, Stanford Comm
+                        </p>
+                    </div>
+                    <div>
+                        <img src="./img/landay.jpeg" alt="James Landay">
+                        <p>
+                            <a href="https://www.landay.org/" target="_blank"><b>James Landay</b></a><br/>
+                                Professor, Stanford CS
+                            </p>
+                    </div>
+                    <div>
+                        <img src="./img/bernstein.jpeg" alt="Michael Bernstein">
+                        <p>
+                            <a href="https://hci.stanford.edu/msb/" target="_blank"><b>Michael Bernstein</b></a><br/>
+                            Assoc Professor, Stanford CS
+                        </p>
+                    </div>
+                </div>
+            </div>
+
+            <div class="card-container">
+                <p class="head_5">ARVA</p>
+                <p class="body">
+                    The <a href="https://avidml.org/arva/" target="_blank">AI Risk and Vulnerability Alliance (ARVA)</a> is a nonprofit organization focused on making AI safer for everyone. Our mission is to empower communities to recognize, diagnose, and manage vulnerabilities in AI that affects them. Our flagship project is the <a href="https://avidml.org" target="_blank">AI Vulnerability Database (AVID)</a>, is an open-source knowledge base of failure modes for AI models, datasets, and systems. 
+                </p>
+            </div>
+
+            <div class="card-container">
+                <p class="head_5">The Team</p>
+                <p class="body">
+                    The IndieLabel Hugging Face deployment was made possible by a wonderful team of volunteers who worked on adapting IndieLabel for use by a general audience, connecting its reports to AVID (AI Vulnerability Database), and deploying it on Hugging Face Spaces. The team includes:
+                </p>
+                <div>
+                    <div class="image-container-wide">
+                        <div class="image-wide">
+                            <img src="./img/lam.jpg" alt="Michelle Lam">
+                            <p>
+                                <a href="http://michelle123lam.github.io" target="_blank"><b>Michelle Lam</b></a><br>
+                                Michelle Lam is a PhD Candidate at Stanford University in the HCI Group. Her research focuses on building systems that empower everyday users to surface their expertise to design and evaluate AI systems.
+                            </p>
+                        </div>
+                        <div class="image-wide">
+                            <img src="./img/anderson.jpeg" alt="Carol Anderson">
+                            <p>
+                                <a href="https://www.carol-anderson.com/" target="_blank"><b>Carol Anderson</b></a><br>
+                                Carol Anderson is a data scientist and machine learning practitioner with expertise in natural language processing (NLP), biological data, and AI ethics. She serves as AVID’s machine learning lead.
+                            </p>
+                        </div>
+                        <div class="image-wide">
+                            <img src="./img/pan.jpg" alt="Christina Pan">
+                            <p>
+                                <a href="https://www.christinaapan.com/" target="_blank"><b>Christina Pan</b></a><br>
+                                Christina Pan started her career building machine learning (ML) models at Google, which inspired her passion for design thinking and AI ethics.
+                            </p>
+                        </div>
+                        <div class="image-wide">
+                            <img src="./img/butters.jpeg" alt="Nathan Butters">
+                            <p>
+                                <b>Nathan Butters</b><br>
+                                Nathan Butters is a product manager in the Office of Ethical and Humane Use at Salesforce. He is a cofounder of the AI Risk and Vulnerability Alliance (ARVA).
+                            </p>
+                        </div>
+                        <div class="image-wide">
+                            <img src="./img/blili_hamelin.jpeg" alt="Borhane Blili-Hamelin">
+                            <p>
+                                <a href="https://borhane.xyz/" target="_blank"><b>Borhane Blili-Hamelin</b></a><br>
+                                Borhane Blili-Hamelin is an ethicist, researcher and AI risk management consultant. He is an officer at the AI Risk and Vulnerability Alliance (ARVA), an affiliate at Data & Society, and a senior consultant at BABL AI. 
+                            </p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<style>
+    a {
+        color: #333;
+        text-decoration: underline;
+    }
+
+    .panel {
+        margin: 0 40px;
+    }
+
+    .blockquote {
+        margin: 0 40px;
+    }
+
+    .head_5 {
+        margin: 10px 0;
+    }
+
+    .card-container {
+        margin: 40px 0;
+    }
+
+    .image-container {
+        margin: 20px 0;
+        display: flex;
+        flex-wrap: wrap;
+        justify-content: center;
+        gap: 20px; /* Adjust the gap between images */
+    }
+    
+    .image-container img {
+        max-width: 110px; /* Adjust the maximum width of images */
+        height: 110px;
+        border-radius: 50%;
+        display: block;
+    }
+
+    .image-container p {
+        text-align: center;
+        margin-top: 5px; /* Adjust the margin between image and name */
+        max-width: 110px;
+        font-size: 11px;
+    }
+
+    .image-container-wide {
+        margin: 20px 0;
+        display: flex;
+        flex-direction: column;
+        justify-content: center;
+        gap: 20px;
+    }
+
+    .image-wide {
+        display: flex;
+        flex-direction: row;
+        align-items: center;
+        gap: 10px;
+    }
+    
+    .image-container-wide img {
+        max-width: 110px; /* Adjust the maximum width of images */
+        height: 110px;
+        border-radius: 50%;
+        display: block;
+    }
+
+    .video-container { 
+        position: relative; 
+        padding-bottom: 10px; 
+        padding-top: 10px; 
+        height: 40vh; 
+        overflow: hidden; 
+    }
+
+    .video-container iframe { 
+        position: absolute; 
+        top: 0; 
+        left: 0; 
+        width: 100%; 
+        height: 100%; 
+    }
+</style>

indie_label_svelte/src/Auditing.svelte

@@ -293,9 +293,6 @@
                 {#if audit_results}
                     <OverallResults
                         data={audit_results}
-                        clusters={clusters}
-                        personalized_model={personalized_model}
-                        cluster={topic}
                     />
                 {/if}
             {:catch error}

indie_label_svelte/src/ClusterResults.svelte

@@ -32,7 +32,7 @@
     export let show_checkboxes = true;
     export let table_width_pct = 80;
     export let rowsPerPage = 10;
-    export let evidence;
+    export let evidence = null;
     export let table_id;
     export let use_model = true;
     export let show_agree_disagree = false;
@@ -232,7 +232,7 @@
         <div class="row">
             <div class="col s8">
                 <VegaLite
-                    {cluster_overview_data}
+                    data={cluster_overview_data}
                     spec={cluster_overview_spec}
                     bind:view={cluster_overview_view}
                 />
@@ -302,7 +302,7 @@
                     <Cell>
                         System<br>decision<br>
                         {#if show_checkboxes}
-                        <span style="font-size:12px; max-width:125px">White = Non-toxic, <br>Grey = Toxic</span>
+                        <span style="font-size:9px; max-width:125px">White = Non-toxic, <br>Grey = Toxic</span>
                         {/if}
                     </Cell>
                     {#if show_num_ratings}
@@ -314,7 +314,7 @@
                     <Cell>
                         Potential error<br>type<br>
                         {#if show_checkboxes}
-                        <span style="font-size:12px; max-width:125px">Darker red = Greater <br>potential system error</span>
+                        <span style="font-size:9px; max-width:125px">Darker red = Greater <br>potential system error</span>
                         {/if}
                     </Cell>
 

indie_label_svelte/src/HypothesisPanel.svelte

@@ -1,10 +1,12 @@
 <script lang="ts">
     import { onMount } from "svelte";
+    import { fade } from 'svelte/transition';
     import ClusterResults from "./ClusterResults.svelte";
     import SubmitReportDialog from "./SubmitReportDialog.svelte";
 
     import Button, { Label } from "@smui/button";
     import Textfield from '@smui/textfield';
+    import Select, { Option } from "@smui/select";
     import { new_evidence } from './stores/new_evidence_store.js';
     import { open_evidence } from './stores/open_evidence_store.js';
     import { topic_chosen } from './stores/cur_topic_store.js';
@@ -22,6 +24,7 @@
     import Checkbox from '@smui/checkbox';
     import FormField from '@smui/form-field';
     import IconButton from "@smui/icon-button";
+    import { Icon } from "@smui/common";
     import Radio from '@smui/radio';
 
     export let model;
@@ -50,6 +53,8 @@
     let editTitle = false;
     let editErrorType = false;
     let unfinished_count = 0;
+    let has_complete_report = false;
+    let save_check_visible = false;  // Whether the save checkmark is visible
     
     function setActive(value: string) {
         selected = value;
@@ -90,7 +95,8 @@
         selected = all_reports[0];
         setActive(selected);
         cur_open_evidence = selected["evidence"];
-        unfinished_count = all_reports.filter(item => !item.complete_status).length
+        unfinished_count = all_reports.filter(item => (item.evidence.length == 0) || (item.text_entry == "") || !(item.sep_selection)).length
+        has_complete_report = hasCompleteReport();
         return all_reports;
     }
 
@@ -129,7 +135,11 @@
 
     let promise_save = Promise.resolve(null);
     function handleSaveReport() {
+        // Briefly display checkmark
         promise_save = saveReport();
+        save_check_visible = true;
+        // Hide save checkmark after 1 second
+        setTimeout(() => save_check_visible = false, 1000);
     }
 
     async function saveReport() {
@@ -143,6 +153,8 @@
         const response = await fetch("./save_reports?" + params);
         const text = await response.text();
         const data = JSON.parse(text);
+        
+        has_complete_report = hasCompleteReport();
         return data;
     }
 
@@ -152,15 +164,14 @@
             error_type: "",
             evidence: [],
             text_entry: "",
-            complete_status: false,
+            sep_selection: "",
         };
         all_reports = all_reports.concat(new_report);
         promise = Promise.resolve(all_reports);
         // Open this new report
         selected = all_reports[all_reports.length - 1];
         cur_open_evidence = selected["evidence"];
-        selected["complete_status"] = false;
-        unfinished_count = all_reports.filter(item => !item.complete_status).length
+        unfinished_count = all_reports.filter(item => (item.evidence.length == 0) || (item.text_entry == "") || !(item.sep_selection)).length
     }
 
     function handleDeleteReport() {
@@ -169,13 +180,11 @@
         promise = Promise.resolve(all_reports);
         selected  = all_reports[0];
         cur_open_evidence = selected["evidence"];
-        unfinished_count = all_reports.filter(item => !item.complete_status).length
+        unfinished_count = all_reports.filter(item => (item.evidence.length == 0) || (item.text_entry == "") || !(item.sep_selection)).length
     }
 
-    function handleMarkComplete() {
-        selected["complete_status"] = !selected["complete_status"];
-        unfinished_count = all_reports.filter(item => !item.complete_status).length
-        handleSaveReport(); // Auto-save report
+    function hasCompleteReport() {
+        return all_reports.some(item => (item.evidence.length > 0) && (item.text_entry != "") && (item.sep_selection));
     }
 
     // Error type
@@ -208,6 +217,14 @@
         editErrorType = false;
 	}
 
+    // SEP selection
+    let all_sep_options = [
+        "Accuracy",
+        "Bias/Discrimination",
+        "Adversarial Example",
+        "Other",
+    ];
+
     let promise_submit = Promise.resolve(null);
     function handleSubmitReport() {
         promise_submit = submitReport();
@@ -231,8 +248,8 @@
         <div class="panel_header">
             <div class="panel_header_content">
                 <div class="page_header">
-                    <img src="/logo.png" style="height: 50px; padding: 0px 20px;" alt="IndieLabel" />
-                    <Button class="user_button" color="secondary" style="margin: 12px 10px;" >
+                    <img src="/logo.png" style="height: 50px;" alt="IndieLabel" />
+                    <Button class="user_button" color="secondary" style="margin: 0 5px; padding: 0 5px;" >
                         <Label>User: {cur_user}</Label>
                     </Button>
                 </div>
@@ -242,7 +259,7 @@
                         on:click={() => (open = !open)}
                         color="primary"
                         disabled={model == null}
-                        style="float: right; padding: 10px; margin-right: 10px;"
+                        style="float: right; padding: 0 5px; margin: 0 5px; max-width: 200px;"
                     >
                         {#if open}
                         <Label>Close</Label>
@@ -284,7 +301,7 @@
                                         on:click={() => setActive(report)}
                                         activated={selected === report}
                                     >   
-                                        {#if report["complete_status"]}
+                                        {#if (report["evidence"].length > 0) && (report["text_entry"] != "") && (report["sep_selection"])}
                                         <Graphic class="material-icons" aria-hidden="true">task_alt</Graphic>
                                         {:else}
                                         <Graphic class="material-icons" aria-hidden="true">radio_button_unchecked</Graphic>
@@ -412,7 +429,7 @@
                                     {/key}
                                 </div>
 
-                                <div class="spacing_vert_60">
+                                <div class="spacing_vert_40">
                                     <div class="head_6">
                                         <b>Summary/Suggestions</b>
                                     </div>
@@ -430,14 +447,17 @@
                                     
                                 </div>
 
-                                <div class="spacing_vert_40">
+                                <div class="spacing_vert_40 spacing_vert_100_bottom">
                                     <div class="head_6">
-                                        <b>Mark report as complete?</b>
-                                        <FormField>
-                                            <Checkbox checked={selected["complete_status"]} on:change={handleMarkComplete} />
-                                        </FormField>
+                                        <b>Audit Category</b>
+                                    </div>
+                                    <div>
+                                        <Select bind:value={selected["sep_selection"]} label="Audit category" style="width: 90%">
+                                            {#each all_sep_options as opt}
+                                                <Option value={opt}>{opt}</Option>
+                                            {/each}
+                                        </Select>
                                     </div>
-                                    
                                 </div>
                             </div>
                             {/if}
@@ -452,55 +472,58 @@
 
         <div class="panel_footer">
             <div class="panel_footer_contents">
+                <!-- New button -->
                 <Button 
                     on:click={handleNewReport} 
                     variant="outlined" 
-                    color="secondary"
-                    style=""
                 >
                     <Label>New</Label>
                 </Button>
 
+                <!-- Delete button -->
                 <!-- <Button 
                     on:click={handleDeleteReport} 
                     variant="outlined" 
-                    color="secondary"
-                    style=""
                 >
                     <Label>Delete</Label>
                 </Button> -->
 
+                <!-- Save button -->
                 <Button 
                     on:click={handleSaveReport} 
                     variant="outlined" 
-                    color="secondary"
                 >
-                    <Label>Save</Label>
+                    <Label>
+                        {#await promise_save}
+                            <CircularProgress style="height: 13.5px; width: 13.5px;" indeterminate />
+                        {:then result}
+                            {#if result && save_check_visible}
+                            <span transition:fade>
+                                <Icon class="material-icons">check</Icon>
+                            </span>
+                            {/if}
+                        {:catch error}
+                            <span style="color: red">{error.message}</span>
+                        {/await}
+                        Save
+                    </Label>
                 </Button>
 
+                <!-- Send to Avid button -->
+                {#key has_complete_report}
                 <Button 
                     on:click={handleSubmitReport} 
                     variant="outlined"
-                    color="secondary"
                 >
-                    <Label>Send Reports</Label>
+                    <Label>Send Reports to AVID</Label>
                 </Button>
+                {/key}
+            </div>
 
-                <div>
-                    <span style="color: grey"><i>Last saved:
-                    {#await promise_save}
-                        <CircularProgress style="height: 32px; width: 32px;" indeterminate />
-                    {:then result}
-                        {#if result}
-                        {new Date().toLocaleTimeString()}
-                        {:else}
-                        —
-                        {/if}
-                    {:catch error}
-                        <p style="color: red">{error.message}</p>
-                    {/await}
-                    </i></span>
-                </div>
+            <div class="feedback_section">
+                <a href="https://forms.gle/vDXchpbBFjDeKjJA6" target="_blank" class="feedback_link">
+                    Share feedback about the tool
+                </a>
             </div>
         </div>
         {/if}
@@ -534,13 +557,23 @@
         background: #f3f3f3;
         z-index: 11;
         bottom: 0;
-        padding: 15px 0px;
+        padding: 5px 0px;
     }
     .panel_footer_contents {
-        /* padding: 0px 20px; */
         display: flex;
         justify-content: space-around;
         align-items: center;
+        padding: 5px 0px 10px 0px;
+    }
+    .feedback_section {
+        display: flex;
+        justify-content: space-around;
+        align-items: center;
+    }
+    .feedback_link {
+        color: var(--mdc-theme-secondary);
+        font-size: 10px;
+        text-decoration: underline;
     }
 
     :global(.mdc-button.user_button) {
@@ -551,18 +584,17 @@
 
     :global(.mdc-button.user_button span) {
         text-overflow: ellipsis;
-        white-space: nowrap;
         overflow: hidden;
     }
 
     .page_header {
         width: 100%;
         background: #e3d6fd;
-        /* padding: 21px 0; */
-        /* border-bottom: 1px solid #e3d6fd; */
         padding: 10.5px 0;
         position: relative;
-        display: inline-block;
+        display: flex;
+        align-items: center;
+        justify-content: space-around;
     }
 
     .page_header:before {
@@ -575,9 +607,11 @@
     }
 
     .hypotheses_header {
-        display: inline-block;
         width: 100%;
         padding: 10px 0;
         vertical-align: middle;
+        display: flex;
+        align-items: center;
+        justify-content: space-around;
     }
 </style>

indie_label_svelte/src/MainPanel.svelte

@@ -1,20 +1,26 @@
 <script lang="ts">
 	import Labeling from "./Labeling.svelte";
 	import Auditing from "./Auditing.svelte";
+	import About from "./About.svelte";
 
 	import Tab, { Label } from "@smui/tab";
 	import TabBar from "@smui/tab-bar";
+	import { Icon } from "@smui/common";
 
-	export let model;
+	export let model = null;
 	export let error_type;
 	export let cur_user;
 
 	// Handle routing
-	let active = "labeling";
+	let active = "about";  // default tab on load
     let searchParams = new URLSearchParams(window.location.search);
     let tab = searchParams.get("tab");
 	if (tab == "auditing") {
 		active = "auditing";
+	} else if (tab == "about") {
+		active = "about";
+	} else if (tab == "labeling") {
+		active = "labeling";
 	}
 	
 </script>
@@ -25,22 +31,31 @@
 
 <div class="auditing_panel">
 	<div class="tab_header">
-		<TabBar tabs={["labeling", "auditing"]} let:tab bind:active>
-			<Tab {tab}>
-				<Label>{tab}</Label>
+		<TabBar tabs={["labeling", "auditing", "about"]} let:tab bind:active>
+			<Tab {tab} minWidth={tab == "about"}>
+				{#if tab == "about"}
+					<Icon class="material-icons">info_outlined</Icon>
+				{:else}
+					<Label>{tab}</Label>
+				{/if}
+				
 			</Tab>
 		</TabBar>
 	</div>
 
 	<div class="panel_contents">
 		<div>
-			<div id="labeling" hidden={active == "auditing"} >
+			<div id="labeling" hidden={active != "labeling"} >
 				<Labeling cur_user={cur_user}/>	
 			</div>
 
-			<div id="auditing" hidden={active == "labeling"} >
+			<div id="auditing" hidden={active != "auditing"} >
 				<Auditing bind:personalized_model={model} bind:cur_error_type={error_type} cur_user={cur_user} on:change/>	
 			</div>
+
+			<div id="about" hidden={active != "about"} >
+				<About />
+			</div>
 		</div>
 
 	</div>

indie_label_svelte/src/ModelPerf.svelte

@@ -32,7 +32,7 @@
     <div>
         <!-- Performance visualization -->
         <div>
-            <VegaLite {perf_plot_data} spec={perf_plot_spec} bind:view={perf_plot_view}/>
+            <VegaLite data={perf_plot_data} spec={perf_plot_spec} bind:view={perf_plot_view}/>
         </div>
     </div>
 </div>

indie_label_svelte/src/OverallResults.svelte

@@ -7,9 +7,6 @@
     import Card, { Content } from '@smui/card';
 
     export let data;
-    export let clusters;
-    export let personalized_model;
-    export let cluster = "";
     
     let show_step1_info = false;
 
@@ -68,7 +65,7 @@
         {/if}
     <div class="row">
         <div class="col s8">
-            <VegaLite {topic_overview_data} spec={topic_overview_spec} bind:view={topic_overview_view}/>
+            <VegaLite data={topic_overview_data} spec={topic_overview_spec} bind:view={topic_overview_view}/>
         </div>
     </div>
 

indie_label_svelte/src/SubmitReportDialog.svelte

@@ -2,20 +2,20 @@
     import Dialog, { Title, Content, Actions } from "@smui/dialog";
     import Button, { Label } from "@smui/button";
     import Textfield from "@smui/textfield";
-    import Select, { Option } from "@smui/select";
     import CircularProgress from '@smui/circular-progress';
+    import Checkbox from '@smui/checkbox';
 
     export let open;
     export let cur_user;
     export let all_reports;
+    let name = "";
     let email = "";
-    let all_sep_options = [
-        "Accuracy",
-        "Bias/Discrimination",
-        "Adversarial Example",
-        "Other",
-    ];
-    let sep_selection = "";
+    // which_reports_to_submit is an array of booleans that tracks whether the report
+    // in the corresponding index of all_reports should be submitted to AVID.
+    let which_reports_to_submit = [];
+    for (let i = 0; i < all_reports.length; i++) {
+        which_reports_to_submit.push(false);
+    }
 
     let promise_submit = Promise.resolve(null);
     function handleSubmitReport() {
@@ -23,11 +23,19 @@
     }
 
     async function submitReport() {
+        //Get the relevant reports
+        let submitted_reports = [];
+        for (let i = 0; i < which_reports_to_submit.length; i++) {
+            if (which_reports_to_submit[i]) {
+                submitted_reports.push(all_reports[i])
+            }
+        }
+
         let req_params = {
             cur_user: cur_user,
-            reports: JSON.stringify(all_reports),
+            reports: JSON.stringify(submitted_reports),
+            name: name,
             email: email,
-            sep_selection: sep_selection,
         };
 
 		let params = new URLSearchParams(req_params).toString();
@@ -51,20 +59,25 @@
             <!-- Description -->
             <div>
                 <b>When you are ready to send all of your audit reports to the <a href="https://avidml.org/" target="_blank">AI Vulnerability Database</a> (AVID), please fill out the following information.</b>
-                Only your submitted reports will be stored in the database for further analysis. While you can submit reports anonymously, we encourage you to provide your email so that we can contact you if we have any questions.
+                Only your submitted reports will be stored in the database for further analysis. While you can submit reports anonymously, we encourage you to provide your name and/or email so that we can contact you if we have any questions.
             </div>
 
             <!-- Summary of complete reports -->
             <div>
-                <p><b>Summary of Reports to Send</b> (Reports that include evidence and are marked as complete)</p>
+                <p><b>Summary of Reports Eligible to Send</b> (Reports that include all fields)</p>
+                <p>    Select the reports you want to submit. </p>
                 <ul>
-                    {#each all_reports as report}
-                        {#if report["complete_status"] && (report["evidence"].length > 0)}
-                            <li>{report["title"]}</li>
+                    {#each all_reports as report, index}
+                        {#if (report["evidence"].length > 0) && (report["text_entry"] != "") && (report["sep_selection"])}
+
+                            <input type="checkbox" bind:checked={which_reports_to_submit[index]} />
+
+                            <span>{report["title"]}</span>
                             <ul>
                                 <li>Error Type: {report["error_type"]}</li>
                                 <li>Evidence: Includes {report["evidence"].length} example{(report["evidence"].length > 1) ? 's' : ''}</li>
                                 <li>Summary/Suggestions: {report["text_entry"]}</li>
+                                <li>Audit Category: {report["sep_selection"] || ''}</li>
                             </ul>
                         {/if}
                     {/each}
@@ -73,11 +86,7 @@
 
             <!-- Form fields -->
             <div>
-                <Select bind:value={sep_selection} label="Audit category" style="width: 90%">
-                    {#each all_sep_options as opt}
-                        <Option value={opt}>{opt}</Option>
-                    {/each}
-                </Select>
+                <Textfield bind:value={name} label="(Optional) Name" style="width: 90%" />
             </div>
             <div>
                 <Textfield bind:value={email} label="(Optional) Contact email" style="width: 90%" />
@@ -85,7 +94,7 @@
 
             <!-- Submission and status message -->
             <div class="dialog_footer">
-                <Button on:click={handleSubmitReport} variant="outlined">
+                <Button on:click={handleSubmitReport} variant="outlined" disabled={which_reports_to_submit.filter(item => item).length == 0}>
                     <Label>Submit Report to AVID</Label>
                 </Button>
 

server.py

@@ -426,6 +426,7 @@ def get_reports():
                     "error_type": "",
                     "evidence": [],
                     "text_entry": "",
+                    "sep_selection": "",
                     "complete_status": False,
                 }
             ]
@@ -446,6 +447,7 @@ def get_fixed_scaffold():
             "error_type": "System is under-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -453,6 +455,7 @@ def get_fixed_scaffold():
             "error_type": "System is over-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -460,6 +463,7 @@ def get_fixed_scaffold():
             "error_type": "System is under-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -467,6 +471,7 @@ def get_fixed_scaffold():
             "error_type": "System is over-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -474,6 +479,7 @@ def get_fixed_scaffold():
             "error_type": "System is under-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
     ]
@@ -484,6 +490,7 @@ def get_empty_report(title, error_type):
             "error_type": error_type,
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         }
 
@@ -494,6 +501,7 @@ def get_tutorial_scaffold():
             "error_type": "System is over-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
     ] 
@@ -606,6 +614,7 @@ def get_prompts_scaffold():
             "error_type": "System is over-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -613,6 +622,7 @@ def get_prompts_scaffold():
             "error_type": "System is under-sensitive",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -620,6 +630,7 @@ def get_prompts_scaffold():
             "error_type": "",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -627,6 +638,7 @@ def get_prompts_scaffold():
             "error_type": "",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
         {
@@ -634,6 +646,7 @@ def get_prompts_scaffold():
             "error_type": "",
             "evidence": [],
             "text_entry": "",
+            "sep_selection": "",
             "complete_status": False,
         },
     ]
@@ -648,7 +661,7 @@ def get_eligible_reports(reports):
 
 # Submit all reports to AVID
 # Logs the responses
-def submit_reports_to_AVID(reports, cur_user, email, sep_selection, debug=DEBUG):
+def submit_reports_to_AVID(reports, cur_user, name, email, debug=DEBUG):
     # Set up the connection to AVID
     root = os.environ.get('AVID_API_URL')
     api_key = os.environ.get('AVID_API_KEY')
@@ -659,7 +672,8 @@ def submit_reports_to_AVID(reports, cur_user, email, sep_selection, debug=DEBUG)
         print("Num eligible reports:", len(reports))
     
     for r in reports:
-        new_report = utils.convert_indie_label_json_to_avid_json(r, cur_user, email, sep_selection)
+        sep_selection = r["sep_selection"]
+        new_report = utils.convert_indie_label_json_to_avid_json(r, cur_user, name, email, sep_selection)
         url = root + "submit"
         response = requests.post(url, json=json.loads(new_report), headers=key) # The loads ensures type compliance
         uuid = response.json()
@@ -693,14 +707,14 @@ def save_reports(debug=DEBUG):
 @app.route("/submit_avid_report")
 def submit_avid_report():
     cur_user = request.args.get("cur_user")
+    name = request.args.get("name")
     email = request.args.get("email")
-    sep_selection = request.args.get("sep_selection")
     reports_json = request.args.get("reports")
 
     reports = json.loads(reports_json)
 
     # Submit reports to AVID
-    submit_reports_to_AVID(reports, cur_user, email, sep_selection)
+    submit_reports_to_AVID(reports, cur_user, name, email)
 
     results = {
         "status": "success",