import gradio as gr
import pandas as pd
import plotly.express as px
from dataclasses import dataclass, field
from typing import List, Dict, Tuple, Union
import json
import os
from collections import OrderedDict
import re
def load_css(css_file_path):
"""Load CSS from a file."""
with open(css_file_path, 'r') as f:
return f.read()
# In the main code:
css = load_css('dashboard.css')
@dataclass
class ScorecardCategory:
name: str
questions: List[Dict[str, Union[str, List[str]]]]
scores: Dict[str, int] = field(default_factory=dict)
def extract_category_number(category_name: str) -> int:
"""Extract the category number from the category name."""
match = re.match(r'^(\d+)\.?\s*.*$', category_name)
return int(match.group(1)) if match else float('inf')
def sort_categories(categories):
"""Sort categories by their numeric prefix."""
return sorted(categories, key=extract_category_number)
# def load_scorecard_templates(directory):
# templates = []
# for filename in os.listdir(directory):
# if filename.endswith('.json'):
# with open(os.path.join(directory, filename), 'r') as file:
# data = json.load(file)
# templates.append(ScorecardCategory(
# name=data['name'],
# questions=data['questions']
# ))
# return templates
def create_category_summary(category_data):
"""Create a summary section for a category"""
# Calculate statistics
total_sections = len(category_data)
completed_sections = sum(1 for section in category_data.values() if section['status'] == 'Yes')
na_sections = sum(1 for section in category_data.values() if section['status'] == 'N/A')
# Calculate completion rates
total_questions = 0
completed_questions = 0
evaluation_types = set()
has_human_eval = False
has_quantitative = False
has_documentation = False
for section in category_data.values():
if section['status'] != 'N/A':
questions = section.get('questions', {})
total_questions += len(questions)
completed_questions += sum(1 for q in questions.values() if q)
# Check for evaluation types
for question in questions.keys():
if 'human' in question.lower():
has_human_eval = True
if any(term in question.lower() for term in ['quantitative', 'metric', 'benchmark']):
has_quantitative = True
if 'documentation' in question.lower():
has_documentation = True
completion_rate = (completed_questions / total_questions * 100) if total_questions > 0 else 0
# Create summary HTML
html = "
"
html += "
📊 Section Summary
"
# Completion metrics
html += "
"
html += "
📈 Completion Metrics
"
html += f"
Overall Completion Rate: {completion_rate:.1f}%
"
html += f"
Sections Completed: {completed_sections}/{total_sections}
"
html += "
"
# Evaluation Coverage
html += "
"
html += "
🎯 Evaluation Coverage
"
html += "
"
html += f"
👥 Human Evaluation
"
html += f"
📊 Quantitative Analysis
"
html += f"
📝 Documentation
"
html += "
"
html += "
"
# Status Breakdown
html += "
"
html += "
📋 Status Breakdown
"
html += create_status_pills(category_data)
html += "
"
html += "
"
return html
def create_overall_summary(model_data, selected_categories):
"""Create a comprehensive summary of all categories"""
scores = model_data['scores']
model_name = model_data['metadata']['Name']
# Initialize counters
total_sections = 0
completed_sections = 0
na_sections = 0
total_questions = 0
completed_questions = 0
# Track evaluation types across all categories
evaluation_types = {
'human': 0,
'quantitative': 0,
'documentation': 0,
'monitoring': 0,
'transparency': 0
}
# Calculate completion rates for categories
category_completion = {}
# Process all categories
for category, category_data in scores.items():
if category not in selected_categories:
continue # Skip unselected categories
category_questions = 0
category_completed = 0
category_na = 0
total_sections_in_category = len(category_data)
na_sections_in_category = sum(1 for section in category_data.values() if section['status'] == 'N/A')
for section in category_data.values():
total_sections += 1
if section['status'] == 'Yes':
completed_sections += 1
elif section['status'] == 'N/A':
na_sections += 1
category_na += 1
if section['status'] != 'N/A':
questions = section.get('questions', {})
section_total = len(questions)
section_completed = sum(1 for q in questions.values() if q)
total_questions += section_total
completed_questions += section_completed
category_questions += section_total
category_completed += section_completed
# Check for evaluation types
for question in questions.keys():
if 'human' in question.lower():
evaluation_types['human'] += 1
if any(term in question.lower() for term in ['quantitative', 'metric', 'benchmark']):
evaluation_types['quantitative'] += 1
if 'documentation' in question.lower():
evaluation_types['documentation'] += 1
if 'monitoring' in question.lower():
evaluation_types['monitoring'] += 1
if 'transparency' in question.lower():
evaluation_types['transparency'] += 1
# Store category information
is_na = na_sections_in_category == total_sections_in_category
completion_rate = (category_completed / category_questions * 100) if category_questions > 0 and not is_na else 0
category_completion[category] = {
'completion_rate': completion_rate,
'is_na': is_na
}
# Create summary HTML
html = ""
html += f"
📊 {model_name} Social Impact Evaluation Summary
"
# Key metrics section
html += "
"
# Overall completion metrics
html += "
"
html += "
📈 Overall Completion
"
completion_rate = (completed_questions / total_questions * 100) if total_questions > 0 else 0
html += f"
Overall Completion Rate: {completion_rate:.1f}%
"
html += f"
Sections Completed: {completed_sections}/{total_sections}
"
html += f"
Questions Completed: {completed_questions}/{total_questions}
"
html += "
"
# Evaluation coverage
html += "
"
html += "
🎯 Evaluation Types Coverage
"
html += "
"
for eval_type, count in evaluation_types.items():
icon = {
'human': '👥',
'quantitative': '📊',
'documentation': '📝',
'monitoring': '📡',
'transparency': '🔍'
}.get(eval_type, '❓')
has_coverage = count > 0
html += f"
{icon} {eval_type.title()}
"
html += "
"
html += "
"
html += "
" # End summary-grid
# Category breakdown
html += "
"
html += "
📋 Category Completion Breakdown
"
html += "
"
# Sort and filter categories
sorted_categories = [cat for cat in sort_categories(scores.keys()) if cat in selected_categories]
for category in sorted_categories:
info = category_completion[category]
category_name = category.split('. ', 1)[1] if '. ' in category else category
# Determine display text and style
if info['is_na']:
completion_text = "N/A"
bar_width = "0"
style_class = "na"
else:
completion_text = f"{info['completion_rate']:.1f}%"
bar_width = f"{info['completion_rate']}"
style_class = "active"
html += f"""
"""
html += "
"
html += "
" # End overall-summary-card
return html
def get_coverage_class(has_feature):
"""Return CSS class based on feature presence"""
return 'covered' if has_feature else 'not-covered'
def create_status_pills(category_data):
"""Create status pill indicators"""
status_counts = {'Yes': 0, 'No': 0, 'N/A': 0}
for section in category_data.values():
status_counts[section['status']] += 1
html = ""
for status, count in status_counts.items():
html += f"
{status}: {count}
"
html += "
"
return html
def get_modality_icon(modality):
"""Return an emoji icon for each modality type."""
icons = {
"Text-to-Text": "📝", # Memo icon for text-to-text
"Text-to-Image": "🎨", # Artist palette for text-to-image
"Image-to-Text": "🔍", # Magnifying glass for image-to-text
"Image-to-Image": "🖼️", # Frame for image-to-image
"Audio": "🎵", # Musical note for audio
"Video": "🎬", # Clapper board for video
"Multimodal": "🔄" # Cycle arrows for multimodal
}
return icons.get(modality, "💫") # Default icon if modality not found
def create_metadata_card(metadata):
"""Create a formatted HTML card for metadata."""
html = ""
return html
def load_models_from_json(directory):
models = {}
for filename in os.listdir(directory):
if filename.endswith('.json'):
with open(os.path.join(directory, filename), 'r') as file:
model_data = json.load(file)
model_name = model_data['metadata']['Name']
models[model_name] = model_data
return OrderedDict(sorted(models.items(), key=lambda x: x[0].lower()))
# Load templates and models
# scorecard_template = load_scorecard_templates('scorecard_templates')
models = load_models_from_json('model_data')
def create_source_html(sources):
if not sources:
return ""
html = ""
for source in sources:
icon = source.get("type", "")
detail = source.get("detail", "")
name = source.get("name", detail)
html += f"
{icon} "
if detail.startswith("http"):
html += f"
{name}"
else:
html += name
html += "
"
html += "
"
return html
def create_leaderboard(selected_categories):
scores = []
for model, data in models.items():
total_score = 0
total_questions = 0
score_by_category = {}
# Calculate scores by category
for category_name, category in data['scores'].items():
category_score = 0
category_total = 0
all_na = True
for section in category.values():
if section['status'] != 'N/A':
all_na = False
questions = section.get('questions', {})
category_score += sum(1 for q in questions.values() if q)
category_total += len(questions)
if category_total > 0:
score_by_category[category_name] = (category_score / category_total) * 100
elif all_na:
score_by_category[category_name] = "N/A"
total_score += category_score
total_questions += category_total
# Calculate overall score
overall_all_na = all(
all(section['status'] == 'N/A' for section in category.values())
for category_name, category in data['scores'].items()
if category_name in selected_categories
)
score_percentage = "N/A" if overall_all_na else (
(total_score / total_questions * 100) if total_questions > 0 else 0
)
# Get model type and URL
model_type = data['metadata'].get('Type', 'Unknown')
model_url = data['metadata'].get('URL', '')
# Get modalities and create badges
modalities = data['metadata'].get('Modalities', [])
modality_badges = " ".join(
f"{get_modality_icon(m)} {m}"
for m in modalities
) if modalities else "💫 Unknown"
# Create model name with HTML link if URL exists
model_display = f'{model}' if model_url else model
# Create entry with numerical scores
model_entry = {
'AI System': model_display,
'Modality': f"{modality_badges}
",
'Overall Completion Rate': score_percentage
}
# Add selected category scores with emojis
category_map = {
'1. Bias, Stereotypes, and Representational Harms Evaluation': '⚖️ Bias and Fairness',
'2. Cultural Values and Sensitive Content Evaluation': '🌍 Cultural Values',
'3. Disparate Performance Evaluation': '📊 Disparate Performance',
'4. Environmental Costs and Carbon Emissions Evaluation': '🌱 Environmental Impact',
'5. Privacy and Data Protection Evaluation': '🔒 Privacy',
'6. Financial Costs Evaluation': '💰 Financial Costs',
'7. Data and Content Moderation Labor Evaluation': '👥 Labor Practices'
}
for full_cat_name, display_name in category_map.items():
if full_cat_name in selected_categories:
score = score_by_category.get(full_cat_name, 0)
model_entry[display_name] = score
scores.append(model_entry)
# Convert to DataFrame
df = pd.DataFrame(scores)
# Sort by Overall Completion Rate descending, putting N/A at the end
df['_sort_value'] = df['Overall Completion Rate'].apply(
lambda x: -float('inf') if x == "N/A" else float(x)
)
df = df.sort_values('_sort_value', ascending=False)
df = df.drop('_sort_value', axis=1)
# Add rank column based on current sort
df.insert(0, 'Rank', range(1, len(df) + 1))
# Get completion rate columns (Overall + category-specific)
completion_rate_columns = ['Overall Completion Rate'] + [
display_name for full_cat_name, display_name in category_map.items()
if full_cat_name in selected_categories
]
# Format non-completion rate columns
df['Rank'] = df['Rank'].astype(str)
# Identify and format highest values for completion rate columns
for col in completion_rate_columns:
if col in df.columns:
# Filter out N/A values to find the maximum numerical value
numeric_values = df[df[col] != "N/A"][col]
if not numeric_values.empty:
max_value = numeric_values.max()
df[col] = df.apply(
lambda row: "N/A" if row[col] == "N/A"
else f"**{row[col]:.1f}%**" if row[col] == max_value
else f"{row[col]:.1f}%",
axis=1
)
else:
df[col] = df[col].apply(lambda x: "N/A")
return df
first_model = next(iter(models.values()))
category_choices = list(first_model['scores'].keys())
with gr.Column(visible=True) as leaderboard_tab:
leaderboard_output = gr.DataFrame(
value=create_leaderboard(category_choices),
interactive=False,
wrap=True,
datatype=["markdown", "markdown", "markdown"] + ["markdown"] * (len(category_choices)+1) # Support markdown in all columns
)
def create_category_chart(selected_models, selected_categories):
if not selected_models:
fig = px.bar(title='Please select at least one model for comparison')
fig.update_layout(showlegend=True) # Ensure legend remains visible
return fig
# Sort categories before processing
selected_categories = sort_categories(selected_categories)
data = []
for model in selected_models:
for category in selected_categories:
if category in models[model]['scores']:
total_score = 0
total_questions = 0
for section in models[model]['scores'][category].values():
if section['status'] != 'N/A':
questions = section.get('questions', {})
total_score += sum(1 for q in questions.values() if q)
total_questions += len(questions)
score_percentage = (total_score / total_questions * 100) if total_questions > 0 else 0
data.append({
'AI System': model,
'Category': category,
'Completion Rate': score_percentage
})
df = pd.DataFrame(data)
if df.empty:
fig = px.bar(title='No data available for the selected models and categories')
else:
fig = px.bar(df, x='AI System', y='Completion Rate', color='Category',
title='AI System Evaluation Completion Rates by Category',
labels={'Completion Rate': 'Completion Rate (%)'},
category_orders={"Category": selected_categories})
fig.update_layout(showlegend=True) # Ensure legend remains visible
return fig
def update_detailed_scorecard(model, selected_categories):
if not model:
return [
gr.update(value="Please select a model to view details.", visible=True),
gr.update(visible=False),
gr.update(visible=False)
]
selected_categories = sort_categories(selected_categories)
metadata_html = create_metadata_card(models[model]['metadata'])
overall_summary_html = create_overall_summary(models[model], selected_categories)
# Combine metadata and overall summary
combined_header = metadata_html + overall_summary_html
total_yes = 0
total_no = 0
total_na = 0
has_non_na = False
# Create category cards
all_cards_content = ""
for category_name in selected_categories:
if category_name in models[model]['scores']:
category_data = models[model]['scores'][category_name]
card_content = f"
{category_name}
"
# Add category-specific summary at the top of each card
card_content += create_category_summary(category_data)
# Sort sections within each category
sorted_sections = sorted(category_data.items(),
key=lambda x: float(re.match(r'^(\d+\.?\d*)', x[0]).group(1)))
category_yes = 0
category_no = 0
category_na = 0
for section, details in sorted_sections:
status = details['status']
if status != 'N/A':
has_non_na = True
sources = details.get('sources', [])
questions = details.get('questions', {})
section_class = "section-na" if status == "N/A" else "section-active"
status_class = status.lower()
status_icon = "●" if status == "Yes" else "○" if status == "N/A" else "×"
card_content += f"
"
card_content += f""
if sources:
card_content += "
"
for source in sources:
icon = source.get("type", "")
detail = source.get("detail", "")
name = source.get("name", detail)
card_content += f"
{icon} "
if detail.startswith("http"):
card_content += f"
{name}"
else:
card_content += name
card_content += "
"
card_content += "
"
if questions:
yes_count = sum(1 for v in questions.values() if v)
total_count = len(questions)
card_content += "
"
if status == "N/A":
card_content += f"View {total_count} N/A items
"
else:
card_content += f"View details ({yes_count}/{total_count} completed)
"
card_content += ""
for question, is_checked in questions.items():
if status == "N/A":
style_class = "na"
icon = "○"
category_na += 1
total_na += 1
else:
if is_checked:
style_class = "checked"
icon = "✓"
category_yes += 1
total_yes += 1
else:
style_class = "unchecked"
icon = "✗"
category_no += 1
total_no += 1
card_content += f"
{icon} {question}
"
card_content += "
"
card_content += "
"
if category_yes + category_no > 0:
category_score = category_yes / (category_yes + category_no) * 100
card_content += f"
Completion Score Breakdown: {category_score:.2f}% Yes: {category_yes}, No: {category_no}, N/A: {category_na}
"
elif category_na > 0:
card_content += f"
Completion Score Breakdown: N/A (All {category_na} items not applicable)
"
card_content += "
"
all_cards_content += card_content
all_cards_content += "
"
# Create total score
if not has_non_na:
total_score_md = "No applicable scores (all items N/A)
"
elif total_yes + total_no > 0:
total_score = total_yes / (total_yes + total_no) * 100
total_score_md = f"Total Score: {total_score:.2f}% (Yes: {total_yes}, No: {total_no}, N/A: {total_na})
"
else:
total_score_md = "No applicable scores (all items N/A)
"
return [
gr.update(value=combined_header, visible=True),
gr.update(value=all_cards_content, visible=True),
gr.update(value=total_score_md, visible=True)
]
first_model = next(iter(models.values()))
category_choices = list(first_model['scores'].keys())
with gr.Blocks(css=css) as demo:
gr.Markdown("# AI System Social Impact Dashboard")
initial_df = create_leaderboard(category_choices)
with gr.Row():
tab_selection = gr.Radio(["Leaderboard", "Category Analysis", "Detailed Scorecard"],
label="Select Tab", value="Leaderboard")
with gr.Row():
model_chooser = gr.Dropdown(choices=[""] + list(models.keys()),
label="Select AI System for Details",
value="",
interactive=True, visible=False)
model_multi_chooser = gr.Dropdown(choices=list(models.keys()),
label="Select AI Systems for Comparison",
value=[],
multiselect=True,
interactive=True,
visible=False,
info="Select one or more AI Systems")
# Category filter now visible for all tabs
category_filter = gr.CheckboxGroup(choices=category_choices,
label="Filter Categories",
value=category_choices)
with gr.Column(visible=True) as leaderboard_tab:
leaderboard_output = gr.DataFrame(
value=initial_df,
interactive=False,
wrap=True,
datatype=["markdown", "markdown", "markdown"] + ["markdown"] * (len(category_choices)+1) # Support markdown in all columns
)
with gr.Column(visible=False) as category_analysis_tab:
# Initialize with empty plot
initial_plot = create_category_chart([], category_choices)
category_chart = gr.Plot(value=initial_plot)
with gr.Column(visible=False) as detailed_scorecard_tab:
model_metadata = gr.HTML()
all_category_cards = gr.HTML()
total_score = gr.Markdown()
def update_dashboard(tab, selected_models, selected_model, selected_categories):
# Default visibility states
component_states = {
"leaderboard": False,
"category_chart": False,
"detailed_scorecard": False,
"model_chooser": False,
"model_multi_chooser": False
}
# Initialize outputs with None
outputs = {
"leaderboard": None,
"category_chart": None,
"model_metadata": None,
"category_cards": None,
"total_score": None
}
# Update visibility based on selected tab
if tab == "Leaderboard":
component_states["leaderboard"] = True
outputs["leaderboard"] = create_leaderboard(selected_categories)
elif tab == "Category Analysis":
component_states["category_chart"] = True
component_states["model_multi_chooser"] = True
if selected_models: # Only update chart if models are selected
outputs["category_chart"] = create_category_chart(selected_models, selected_categories)
elif tab == "Detailed Scorecard":
component_states["detailed_scorecard"] = True
component_states["model_chooser"] = True
if selected_model:
scorecard_updates = update_detailed_scorecard(selected_model, selected_categories)
outputs["model_metadata"] = scorecard_updates[0]
outputs["category_cards"] = scorecard_updates[1]
outputs["total_score"] = scorecard_updates[2]
# Return updates in the correct order
return [
gr.update(visible=component_states["leaderboard"]),
gr.update(visible=component_states["category_chart"]),
gr.update(visible=component_states["detailed_scorecard"]),
gr.update(visible=component_states["model_chooser"]),
gr.update(visible=component_states["model_multi_chooser"]),
outputs["leaderboard"] if outputs["leaderboard"] is not None else gr.update(),
outputs["category_chart"] if outputs["category_chart"] is not None else gr.update(),
outputs["model_metadata"] if outputs["model_metadata"] is not None else gr.update(),
outputs["category_cards"] if outputs["category_cards"] is not None else gr.update(),
outputs["total_score"] if outputs["total_score"] is not None else gr.update()
]
# Set up event handlers
for component in [tab_selection, model_chooser, model_multi_chooser, category_filter]:
component.change(
fn=update_dashboard,
inputs=[tab_selection, model_multi_chooser, model_chooser, category_filter],
outputs=[leaderboard_tab, category_analysis_tab, detailed_scorecard_tab,
model_chooser, model_multi_chooser,
leaderboard_output, category_chart, model_metadata,
all_category_cards, total_score]
)
# Launch the app
if __name__ == "__main__":
demo.launch(ssr_mode=False)