| import { React, useState, useRef, useEffect, forwardRef } from "react"; |
| import { set, useForm } from "react-hook-form"; |
| import { useReactToPrint } from "react-to-print"; |
| import styled from "styled-components"; |
| import axios from "axios"; |
| import Button from "../styled_components/Button"; |
| import { ProgressBar } from "primereact/progressbar"; |
| import "primereact/resources/themes/lara-light-cyan/theme.css"; |
| import Nav from "../styled_components/Nav"; |
| import { InputText } from "primereact/inputtext"; |
| import { Chart } from "primereact/chart"; |
| import WER_GenderChart from "../styled_components/WER_GenderChart"; |
| import { useContext } from "react"; |
| import { DataContext } from "../DataContext"; |
| import BoxPlot from "../styled_components/Box_plot"; |
| import Speedometer from "../styled_components/Speedometer"; |
| import { Toast } from "primereact/toast"; |
| import { ScrollTop } from "primereact/scrolltop"; |
| import ScoreCard from "../styled_components/Scorecard"; |
| import { media } from "../Utils/helper"; |
| import StopAudit from "../styled_components/StopAudit"; |
|
|
| export function Request({ showSucess, showError, showInfo, baseUrl }) { |
| let [result, setResult] = useState(false); |
| let [isRunning, setIsRunning] = useState(false); |
| let [category_FS, setCategory_FS] = useState([]); |
| let [auditProgress, setAuditProgress] = useState(0); |
| const { sharedData, setSharedData } = useContext(DataContext); |
| let [fetchProgress, setFetchProgress] = useState(false); |
|
|
| const contentRef = useRef(null); |
| const reactToPrintFn = useReactToPrint({ contentRef }); |
|
|
| const [isUpdated, setIsUpdated] = useState(false); |
|
|
| const getAuditProgress = async () => { |
| try { |
| const response = await axios.get(`${baseUrl}/status`); |
| setAuditProgress(Math.round(response.data["%_completed"] * 100) / 100); |
| } catch (error) { |
| console.error("Error fetching audit progress:", error); |
| } |
| }; |
|
|
| const sendPostRequest = async (postData) => { |
| try { |
| const response = await axios.post(`${baseUrl}/insert`, postData); |
|
|
| if (response.data.exists) { |
| console.log("Already in DB"); |
| showInfo(); |
| } else { |
| console.log("Success:", response.data); |
| showSucess(); |
| } |
| } catch (error) { |
| console.error("Error:", error); |
| showError(); |
| } |
| }; |
|
|
| const publishResult = () => { |
| const postData = { |
| Model: result["ASR_model"], |
| WER: Math.round(result["Avg_wer"] * 100) / 100, |
| RTFX: Math.round(result["Avg_rtfx"] * 100) / 100, |
| FAAS: Math.round(result["FAAS"] * 100) / 100, |
| FS: Math.round(result["Overall Fairness Score"] * 100) / 100, |
| FS_G: |
| Math.round(result["Adjusted Category Fairness Score"]["gender"] * 100) / |
| 100, |
| FS_L: |
| Math.round( |
| result["Adjusted Category Fairness Score"]["first_language"] * 100 |
| ) / 100, |
| FS_SEG: |
| Math.round( |
| result["Adjusted Category Fairness Score"]["socioeconomic_bkgd"] * 100 |
| ) / 100, |
| FS_E: |
| Math.round( |
| result["Adjusted Category Fairness Score"]["ethnicity"] * 100 |
| ) / 100, |
| }; |
| sendPostRequest(postData); |
| }; |
|
|
| const { |
| register, |
| handleSubmit, |
| formState: { errors }, |
| clearErrors, |
| trigger, |
| setValue, |
| } = useForm(); |
| const onSubmit = async (data) => { |
| setIsRunning(true); |
| setFetchProgress(true); |
| const queryString = new URLSearchParams(data).toString(); |
|
|
| try { |
| const res = await axios.get(`${baseUrl}/api?${queryString}`, { |
| headers, |
| timeout: 10800000, |
| }); |
| setResult(res.data); |
| setIsRunning(false); |
| setCategory_FS([ |
| { |
| category: "Gender", |
| FS: result["Adjusted Category Fairness Score"]?.["gender"], |
| }, |
| { |
| category: "Language", |
| FS: result["Adjusted Category Fairness Score"]?.["first_language"], |
| }, |
| { |
| category: "Ethnicity", |
| FS: result["Adjusted Category Fairness Score"]?.["ethnicity"], |
| }, |
| { |
| category: "Socioeconomic BG", |
| FS: result["Adjusted Category Fairness Score"]?.[ |
| "socioeconomic_bkgd" |
| ], |
| }, |
| ]); |
| setSharedData({ |
| ...sharedData, |
| Request: res.data, |
| Category_FS: [ |
| { |
| category: "Gender", |
| FS: result["Adjusted Category Fairness Score"]?.["gender"], |
| }, |
| { |
| category: "Language", |
| FS: result["Adjusted Category Fairness Score"]?.["first_language"], |
| }, |
| { |
| category: "Ethnicity", |
| FS: result["Adjusted Category Fairness Score"]?.["ethnicity"], |
| }, |
| { |
| category: "Socioeconomic BG", |
| FS: result["Adjusted Category Fairness Score"]?.[ |
| "socioeconomic_bkgd" |
| ], |
| }, |
| ], |
| }); |
|
|
| } catch (error) { |
| console.error("Retrying due to error:", error.message); |
| } |
| }; |
|
|
| useEffect(() => { |
| let interval; |
|
|
| if (fetchProgress) { |
| interval = setInterval(() => { |
| getAuditProgress(); |
| }, 3000); |
| } |
|
|
| |
| return () => { |
| if (interval) clearInterval(interval); |
| }; |
| }, [isRunning]); |
|
|
| useEffect(() => { |
| if (sharedData && sharedData.Request) { |
| setResult(sharedData.Request); |
| } |
| if (sharedData && sharedData.Category_FS) { |
| setCategory_FS(sharedData.Category_FS); |
| } |
| if (result && result["Adjusted Category Fairness Score"]) { |
| setCategory_FS([ |
| { |
| category: "Gender", |
| FS: result["Adjusted Category Fairness Score"]["gender"], |
| }, |
| { |
| category: "Language", |
| FS: result["Adjusted Category Fairness Score"]["first_language"], |
| }, |
| { |
| category: "Ethnicity", |
| FS: result["Adjusted Category Fairness Score"]["ethnicity"], |
| }, |
| { |
| category: "Socioeconomic BG", |
| FS: result["Adjusted Category Fairness Score"]["socioeconomic_bkgd"], |
| }, |
| ]); |
| } |
| }, [result]); |
|
|
| const headers = { |
| "ngrok-skip-browser-warning": "10008", |
| }; |
|
|
| return (<> |
| |
| <Main> |
| <PrintContainer ref={contentRef}> |
| <h2>Check ASR Model Fairness Score</h2> |
| <InfoPanel> |
| <h4>How the Fairness Audit Works:</h4> |
| <ol> |
| <li> |
| Enter a Hugging Face ASR model ID (e.g.,{" "} |
| <ModelExample>facebook/wav2vec2-base-960h</ModelExample>) |
| </li> |
| <li> |
| Our system will run inference using a 10% stratified sample from |
| Meta's FairSpeech dataset |
| </li> |
| <li> |
| Results will show performance differences across demographic |
| groups including gender, language, ethnicity, and socioeconomic |
| background |
| </li> |
| </ol> |
| <Tip> |
| <i className="pi pi-info-circle"></i> The audit may take up to 60 |
| minutes to complete depending on the model size and hardware. |
| </Tip> |
| <Tip> |
| <i className="pi pi-info-circle"></i> If the model has been |
| evaluated before, results appear instantly; otherwise, please wait |
| for the audit to finish. |
| </Tip> |
| <Tip> |
| <i className="pi pi-info-circle"></i> Please ensure that the ASR model you are requesting for audit is available for use within the Hugging Face Transformers library. |
| </Tip> |
| {/* <Tip> |
| <i className="pi pi-info-circle"></i> If audit for a model is in progress then please wait for it to finish, |
| then run fairness audit for your model. |
| </Tip> */} |
| </InfoPanel> |
| |
| <form onSubmit={handleSubmit(onSubmit)}> |
| <FormGroup> |
| <label htmlFor="username">Hugging Face Model ID:</label> |
| <ModelInputWrapper> |
| <InputText |
| placeholder="Example: facebook/wav2vec2-base-960h" |
| variant="filled" |
| id="username" |
| {...register("ASR_model", { |
| required: "Please enter a valid model path", |
| maxLength: { |
| value: 96, |
| message: "Model ID must be at most 96 characters long", |
| }, |
| pattern: { |
| value: /^[a-zA-Z0-9/_\-.]+$/, |
| message: "Model ID can only contain letters, digits, '/ ', '_', and '-'" |
| }, |
| validate: { |
| noDoubleDashDot: v => |
| !/--|\.\./.test(v) || "Model ID must not contain '--' or '..'", |
| noEdgeDashDot: v => |
| !/^[-.]|[-.]$/.test(v) || "Model ID must not start or end with '-' or '.'" |
| } |
| })} |
| /> |
| </ModelInputWrapper> |
| {errors.ASR_model && ( |
| <ErrorMessage>{errors.ASR_model.message}</ErrorMessage> |
| )} |
| |
| <ExampleModels> |
| <span>Try these examples: </span> |
| <ExampleButton |
| type="button" |
| onClick={() => |
| setValue("ASR_model", "facebook/wav2vec2-base-960h", { |
| shouldValidate: true, |
| shouldDirty: true, |
| }) |
| } |
| > |
| wav2vec2 |
| </ExampleButton> |
| <ExampleButton |
| type="button" |
| onClick={() => |
| setValue("ASR_model", "openai/whisper-tiny", { |
| shouldValidate: true, |
| shouldDirty: true, |
| }) |
| } |
| > |
| whisper-tiny |
| </ExampleButton> |
| <ExampleButton |
| type="button" |
| onClick={() => |
| setValue("ASR_model", "openai/whisper-small", { |
| shouldValidate: true, |
| shouldDirty: true, |
| }) |
| } |
| > |
| whisper-small |
| </ExampleButton> |
| </ExampleModels> |
| </FormGroup> |
| <BtnGroup> |
| <Button |
| type="submit" |
| shadow="blue" |
| bg="#3b82f6" |
| color="white" |
| disabled={isRunning} |
| > |
| Run Fairness Audit |
| </Button> |
| <StopAudit baseUrl={baseUrl} /> |
| </BtnGroup> |
| </form> |
| |
| {result.message && ( |
| <h3 style={{ marginBottom: 0 }}> |
| <i> |
| <u>{result.message}</u> |
| </i> |
| <br /> |
| {result.status && ( |
| <><b> |
| Progress :{" "} |
| {Math.round(result.status["%_completed"] * 100) / 100} % </b> <I>( * at the time of request )</I> |
| </> |
| )} |
| </h3> |
| )} |
| |
| {isRunning ? ( |
| <> |
| <br /> |
| <ProgressBar |
| color="#3b82f6" |
| mode="indeterminate" |
| style={{ height: "15px", borderRadius: "1000px" }} |
| /> |
| </> |
| ) : ( |
| auditProgress > 0 && |
| auditProgress < 100 && |
| !result.ASR_model && ( |
| <> |
| <ProgressBar |
| color="#3b82f6" |
| value={auditProgress} |
| style={{ |
| height: "15px", |
| fontSize: "13px", |
| borderRadius: "9000px", |
| marginBlock: "1.2rem", |
| }} |
| /> |
| </> |
| ) |
| )} |
|
|
| {result.ASR_model && ( |
| <> |
| {" "} |
| <br></br> |
| <GraphContainer> |
| <PageBreakContainer> |
| <Head>Summary</Head> |
| <br></br> |
| <Summary_wrap> |
| <div> |
| <ScoreCard |
| label="Fairness-Adjusted ASR Score (FAAS) : " |
| width={"100%"} |
| score={result["FAAS"]} |
| fontWeight={700} |
| ></ScoreCard> |
| </div> |
| <Summary_Chart> |
| <Speedometer |
| value={result["Overall Fairness Score"]} |
| ></Speedometer> |
| <WER_GenderChart |
| graphData={category_FS} |
| title="Fairness Score (By Category)" |
| labelY="Fairness Score" |
| xAxis="category" |
| yAxis="FS" |
| angle={-45} |
| bMargin={80} |
| height="400px" |
| colorStroke="rgb(2, 167, 5)" |
| colorFill="rgba(24, 219, 27, 0.447)" |
| /> |
| </Summary_Chart> |
| </Summary_wrap> |
| <br></br> |
| <br></br> |
| <hr></hr> |
| </PageBreakContainer> |
| |
| <PageBreakContainer> |
| <Head>Gender</Head> |
| <br></br> |
| <Box_chartV> |
| <ScoreCard |
| label="Fairness Score : " |
| width={"100%"} |
| score={result["Adjusted Category Fairness Score"]["gender"]} |
| ></ScoreCard> |
| <BoxPlot werData={result.wer_Gender} title="Gender"></BoxPlot> |
| </Box_chartV> |
| </PageBreakContainer> |
| |
| <PageBreakContainer> |
| <Head>Socioeconomic Background</Head> |
| <br></br> |
| <Box_chartV> |
| <ScoreCard |
| label="Fairness Score : " |
| width={"100%"} |
| score={ |
| result["Adjusted Category Fairness Score"][ |
| "socioeconomic_bkgd" |
| ] |
| } |
| ></ScoreCard> |
| <BoxPlot |
| werData={result.wer_SEG} |
| title="Socioeconomic Background" |
| ></BoxPlot> |
| </Box_chartV> |
| </PageBreakContainer> |
| |
| <PageBreakContainer> |
| <Head>Language</Head> |
| <br></br> |
| <Box_chartV> |
| <ScoreCard |
| label="Fairness Score : " |
| width={"100%"} |
| score={ |
| result["Adjusted Category Fairness Score"][ |
| "first_language" |
| ] |
| } |
| ></ScoreCard> |
| <BoxPlot |
| werData={result.wer_Language} |
| title="Language" |
| mb={80} |
| ></BoxPlot> |
| </Box_chartV> |
| </PageBreakContainer> |
| |
| <PageBreakContainer> |
| <Head>Ethnicity</Head> |
| <br></br> |
| <Box_chartV> |
| <ScoreCard |
| label="Fairness Score : " |
| width={"100%"} |
| score={ |
| result["Adjusted Category Fairness Score"]["ethnicity"] |
| } |
| ></ScoreCard> |
| <BoxPlot |
| werData={result.wer_Ethnicity} |
| title="Ethnicity" |
| mb={165} |
| ></BoxPlot> |
| </Box_chartV> |
| </PageBreakContainer> |
| </GraphContainer> |
| <BtnGroup> |
| <Button |
| onClick={publishResult} |
| shadow="blue" |
| bg="#3b82f6" |
| color="white" |
| > |
| Publish to Leaderboard |
| </Button> |
| <Button |
| shadow="blue" |
| bg="#3b82f6" |
| color="white" |
| onClick={() => reactToPrintFn()} |
| > |
| Export PDF |
| </Button> |
| </BtnGroup> |
| <ScrollTop /> |
| </> |
| )} |
| </PrintContainer> |
| </Main> |
| </> |
| ); |
| } |
|
|
| const Main = styled.div` |
| /* .p-progressbar-value { |
| background-color: #3b82f6; |
| } */ |
| `; |
|
|
| const I = styled.i` |
| font-size: 0.8rem; |
| color: #979797; |
| font-weight: 100; |
| ` |
|
|
| const PrintContainer = styled.div` |
| @media print { |
| margin: 1.5rem; /* Add 1-inch margin on all sides */ |
| margin-top: 1rem !important; |
| /* padding: 0.5in; Add padding inside the content */ |
| box-sizing: border-box; |
| } |
| `; |
|
|
| |
| const InfoPanel = styled.div` |
| background-color: #f0f7ff; |
| border-left: 4px solid #3b82f6; |
| padding: 1rem 1.5rem; |
| margin-bottom: 1.5rem; |
| border-radius: 0.25rem; |
| |
| h4 { |
| margin-top: 0; |
| margin-bottom: 0.75rem; |
| font-size: 1.1rem; |
| font-weight: 600; |
| } |
| |
| ol { |
| margin-left: 1.5rem; |
| margin-bottom: 0.75rem; |
| @media ${media.mobile} { |
| margin-left: 0.5rem; |
| margin-bottom: 0.5rem; |
| padding-left: 0.9rem; |
| } |
| } |
| |
| li { |
| margin-bottom: 0.5rem; |
| } |
| `; |
|
|
| const Tip = styled.p` |
| font-size: 1rem; |
| margin-top: 0.5rem; |
| color: #3b82f6; |
| font-style: italic; |
| |
| i { |
| margin-right: 0.5rem; |
| } |
| @media ${media.tablet} { |
| font-size: 0.9rem; |
| margin-top: 0.3rem; |
| } |
| @media ${media.mobile} { |
| font-size: 0.8rem; |
| margin-top: 0.3rem; |
| } |
| |
| `; |
|
|
| const ModelExample = styled.code` |
| background: rgba(0, 0, 0, 0.05); |
| padding: 0.2rem 0.4rem; |
| border-radius: 3px; |
| font-family: monospace; |
| font-size: 0.9em; |
| `; |
|
|
| const FormGroup = styled.div` |
| margin-bottom: 1rem; |
| |
| label { |
| display: block; |
| margin-bottom: 0.5rem; |
| font-weight: 500; |
| align-items: center; |
| justify-content: center; |
| } |
| `; |
|
|
| const ModelInputWrapper = styled.div` |
| display: flex; |
| align-items: center; |
| margin-bottom: 0.5rem; |
| input { |
| margin-bottom: 0.5rem !important; |
| font-family: "Poppins", monospace; |
| width: 60%; |
| @media ${media.tablet} { |
| width: 80%; |
| } |
| @media ${media.mobile} { |
| width: 95%; |
| } |
| } |
| `; |
|
|
| const ModelPrefix = styled.div` |
| background-color: #f3f4f6; |
| padding: 0.6rem 0.75rem; |
| border: 1px solid #d1d5db; |
| border-right: none; |
| border-radius: 0.25rem 0 0 0.25rem; |
| color: #6b7280; |
| font-family: monospace; |
| `; |
|
|
| const ErrorMessage = styled.div` |
| color: #ef4444; |
| font-size: 0.875rem; |
| margin-top: 0.25rem; |
| margin-bottom: 0.6rem; |
| `; |
|
|
| const ExampleModels = styled.div` |
| margin-bottom: 1rem; |
| display: flex; |
| align-items: center; |
| flex-wrap: wrap; |
| gap: 0.5rem; |
| |
| span { |
| color: #6b7280; |
| font-size: 0.875rem; |
| } |
| `; |
|
|
| const ExampleButton = styled.button` |
| background: none; |
| border: 1px solid #d1d5db; |
| border-radius: 0.25rem; |
| padding: 0.25rem 0.5rem; |
| font-size: 0.75rem; |
| cursor: pointer; |
| color: #374151; |
| |
| &:hover { |
| background-color: #f3f4f6; |
| } |
| `; |
|
|
| |
| const PageBreakContainer = styled.div` |
| page-break-inside: avoid; /* Prevent breaking inside this container */ |
| break-inside: avoid; /* For modern browsers */ |
| margin-bottom: 2rem; /* Add spacing between sections */ |
| `; |
|
|
| |
| export const Head = styled.h6` |
| font-size: 2rem; |
| color: #3b82f6; |
| margin-block: 0; |
| text-decoration: 4px solid underline; |
| `; |
|
|
| const GraphContainer = styled.div` |
| display: flex; |
| flex-direction: column; |
| gap: 2.5rem; |
| `; |
|
|
| const ComboDivG = styled.div` |
| flex-grow: 1; |
| `; |
| const ComboDivS = styled.div` |
| flex-grow: 1.5; |
| `; |
|
|
| const Combo_Chart = styled.div` |
| display: flex; |
| gap: 2rem; |
| justify-content: space-between; |
| `; |
|
|
| const Summary_wrap = styled.div` |
| display: flex; |
| flex-direction: column; |
| `; |
| const Summary_Chart = styled.div` |
| display: flex; |
| gap: 2rem; |
| min-height: 60vh; |
| justify-content: space-between; |
| align-content: center; |
| align-items: center; |
| flex-direction: column; |
| @media (min-width: 1028px) { |
| flex-direction: row; |
| } |
| `; |
|
|
| const Box_chartV = styled.div` |
| display: flex; |
| flex-direction: column; |
| min-height: 60vh; |
| justify-content: space-between; |
| align-content: center; |
| align-items: center; |
| `; |
|
|
| const BtnGroup = styled.div` |
| display: flex; |
| flex-wrap: wrap; |
| gap: 2rem; |
| /* margin-top: 1rem; */ |
| `; |
|
|