import gradio as gr import pandas as pd from dataclasses import dataclass, field from typing import List, Dict, Tuple, Union import json import os from collections import OrderedDict import re import plotly.graph_objects as go import plotly.express as px # from plotly.subplots import make_subplots # import math def load_css(css_file_path): """Load CSS from a file.""" with open(css_file_path, 'r') as f: return f.read() # In the main code: css = load_css('dashboard.css') @dataclass class ScorecardCategory: name: str questions: List[Dict[str, Union[str, List[str]]]] scores: Dict[str, int] = field(default_factory=dict) def extract_category_number(category_name: str) -> int: """Extract the category number from the category name.""" match = re.match(r'^(\d+)\.?\s*.*$', category_name) return int(match.group(1)) if match else float('inf') def sort_categories(categories): """Sort categories by their numeric prefix.""" return sorted(categories, key=extract_category_number) # def load_scorecard_templates(directory): # templates = [] # for filename in os.listdir(directory): # if filename.endswith('.json'): # with open(os.path.join(directory, filename), 'r') as file: # data = json.load(file) # templates.append(ScorecardCategory( # name=data['name'], # questions=data['questions'] # )) # return templates def create_category_summary(category_data): """Create a summary section for a category""" # Calculate statistics total_sections = len(category_data) completed_sections = sum(1 for section in category_data.values() if section['status'] == 'Yes') na_sections = sum(1 for section in category_data.values() if section['status'] == 'N/A') # Calculate completion rates total_questions = 0 completed_questions = 0 evaluation_types = set() has_human_eval = False has_quantitative = False has_documentation = False for section in category_data.values(): if section['status'] != 'N/A': questions = section.get('questions', {}) total_questions += len(questions) completed_questions += sum(1 for q in questions.values() if q) # Check for evaluation types for question in questions.keys(): if 'human' in question.lower(): has_human_eval = True if any(term in question.lower() for term in ['quantitative', 'metric', 'benchmark']): has_quantitative = True if 'documentation' in question.lower(): has_documentation = True completion_rate = (completed_questions / total_questions * 100) if total_questions > 0 else 0 # Create summary HTML html = "
" html += "
📊 Section Summary
" # Completion metrics html += "
" html += "
📈 Completion Metrics
" html += f"
Overall Completion Rate: {completion_rate:.1f}%
" html += f"
Sections Completed: {completed_sections}/{total_sections}
" html += "
" # Evaluation Coverage html += "
" html += "
🎯 Evaluation Coverage
" html += "
" html += f"
👥 Human Evaluation
" html += f"
📊 Quantitative Analysis
" html += f"
📝 Documentation
" html += "
" html += "
" # Status Breakdown html += "
" html += "
📋 Status Breakdown
" html += create_status_pills(category_data) html += "
" html += "
" return html def create_overall_summary(model_data, selected_categories): """Create a comprehensive summary of all categories""" scores = model_data['scores'] model_name = model_data['metadata']['Name'] # Initialize counters total_sections = 0 completed_sections = 0 na_sections = 0 total_questions = 0 completed_questions = 0 # Track evaluation types across all categories evaluation_types = { 'human': 0, 'quantitative': 0, 'documentation': 0, 'monitoring': 0, 'transparency': 0 } # Calculate completion rates for categories category_completion = {} # Process all categories for category, category_data in scores.items(): if category not in selected_categories: continue # Skip unselected categories category_questions = 0 category_completed = 0 category_na = 0 total_sections_in_category = len(category_data) na_sections_in_category = sum(1 for section in category_data.values() if section['status'] == 'N/A') for section in category_data.values(): total_sections += 1 if section['status'] == 'Yes': completed_sections += 1 elif section['status'] == 'N/A': na_sections += 1 category_na += 1 if section['status'] != 'N/A': questions = section.get('questions', {}) section_total = len(questions) section_completed = sum(1 for q in questions.values() if q) total_questions += section_total completed_questions += section_completed category_questions += section_total category_completed += section_completed # Check for evaluation types for question in questions.keys(): if 'human' in question.lower(): evaluation_types['human'] += 1 if any(term in question.lower() for term in ['quantitative', 'metric', 'benchmark']): evaluation_types['quantitative'] += 1 if 'documentation' in question.lower(): evaluation_types['documentation'] += 1 if 'monitoring' in question.lower(): evaluation_types['monitoring'] += 1 if 'transparency' in question.lower(): evaluation_types['transparency'] += 1 # Store category information is_na = na_sections_in_category == total_sections_in_category completion_rate = (category_completed / category_questions * 100) if category_questions > 0 and not is_na else 0 category_completion[category] = { 'completion_rate': completion_rate, 'is_na': is_na } # Create summary HTML html = "
" html += f"
📊 {model_name} Social Impact Evaluation Summary
" # Key metrics section html += "
" # Overall completion metrics html += "
" html += "
📈 Overall Completion
" completion_rate = (completed_questions / total_questions * 100) if total_questions > 0 else 0 html += f"
Overall Completion Rate: {completion_rate:.1f}%
" html += f"
Sections Completed: {completed_sections}/{total_sections}
" html += f"
Questions Completed: {completed_questions}/{total_questions}
" html += "
" # Evaluation coverage html += "
" html += "
🎯 Evaluation Types Coverage
" html += "
" for eval_type, count in evaluation_types.items(): icon = { 'human': '👥', 'quantitative': '📊', 'documentation': '📝', 'monitoring': '📡', 'transparency': '🔍' }.get(eval_type, '❓') has_coverage = count > 0 html += f"
{icon} {eval_type.title()}
" html += "
" html += "
" html += "
" # End summary-grid # Category breakdown html += "
" html += "
📋 Category Completion Breakdown
" html += "
" # Sort and filter categories sorted_categories = [cat for cat in sort_categories(scores.keys()) if cat in selected_categories] for category in sorted_categories: info = category_completion[category] category_name = category.split('. ', 1)[1] if '. ' in category else category # Determine display text and style if info['is_na']: completion_text = "N/A" bar_width = "0" style_class = "na" else: completion_text = f"{info['completion_rate']:.1f}%" bar_width = f"{info['completion_rate']}" style_class = "active" html += f"""
{category_name}
{completion_text}
""" html += "
" html += "
" # End overall-summary-card return html def get_coverage_class(has_feature): """Return CSS class based on feature presence""" return 'covered' if has_feature else 'not-covered' def create_status_pills(category_data): """Create status pill indicators""" status_counts = {'Yes': 0, 'No': 0, 'N/A': 0} for section in category_data.values(): status_counts[section['status']] += 1 html = "
" for status, count in status_counts.items(): html += f"
{status}: {count}
" html += "
" return html def get_modality_icon(modality): """Return an emoji icon for each modality type.""" icons = { "Text-to-Text": "📝", # Memo icon for text-to-text "Text-to-Image": "🎨", # Artist palette for text-to-image "Image-to-Text": "🔍", # Magnifying glass for image-to-text "Image-to-Image": "🖼️", # Frame for image-to-image "Audio": "🎵", # Musical note for audio "Video": "🎬", # Clapper board for video "Multimodal": "🔄" # Cycle arrows for multimodal } return icons.get(modality, "💫") # Default icon if modality not found def create_metadata_card(metadata): """Create a formatted HTML card for metadata.""" html = "
" html += "
AI System Information
" html += "
" # Handle special formatting for modalities modalities = metadata.get("Modalities", []) formatted_modalities = "" if modalities: formatted_modalities = " ".join( f"{get_modality_icon(m)} {m}" for m in modalities ) # Order of metadata display (customize as needed) display_order = ["Name", "Provider", "Type", "URL"] # Display ordered metadata first for key in display_order: if key in metadata: value = metadata[key] if key == "URL": html += f"
{key}: " html += f"{value}
" else: html += f"
{key}: {value}
" # Add modalities if present if formatted_modalities: html += f"
Modalities:
{formatted_modalities}
" # Add any remaining metadata not in display_order for key, value in metadata.items(): if key not in display_order and key != "Modalities": html += f"
{key}: {value}
" html += "
" return html def load_models_from_json(directory): models = {} for filename in os.listdir(directory): if filename.endswith('.json'): with open(os.path.join(directory, filename), 'r') as file: model_data = json.load(file) model_name = model_data['metadata']['Name'] models[model_name] = model_data return OrderedDict(sorted(models.items(), key=lambda x: x[0].lower())) # Load templates and models # scorecard_template = load_scorecard_templates('scorecard_templates') models = load_models_from_json('model_data') def create_source_html(sources): if not sources: return "" html = "
" for source in sources: icon = source.get("type", "") detail = source.get("detail", "") name = source.get("name", detail) html += f"
{icon} " if detail.startswith("http"): html += f"{name}" else: html += name html += "
" html += "
" return html def create_leaderboard(selected_categories): scores = [] for model, data in models.items(): total_score = 0 total_questions = 0 score_by_category = {} # Calculate scores by category for category_name, category in data['scores'].items(): category_score = 0 category_total = 0 all_na = True for section in category.values(): if section['status'] != 'N/A': all_na = False questions = section.get('questions', {}) category_score += sum(1 for q in questions.values() if q) category_total += len(questions) if category_total > 0: score_by_category[category_name] = (category_score / category_total) * 100 elif all_na: score_by_category[category_name] = "N/A" total_score += category_score total_questions += category_total # Calculate overall score overall_all_na = all( all(section['status'] == 'N/A' for section in category.values()) for category_name, category in data['scores'].items() if category_name in selected_categories ) score_percentage = "N/A" if overall_all_na else ( (total_score / total_questions * 100) if total_questions > 0 else 0 ) # Get model type and URL model_type = data['metadata'].get('Type', 'Unknown') model_url = data['metadata'].get('URL', '') # Get modalities and create badges modalities = data['metadata'].get('Modalities', []) modality_badges = " ".join( f"{get_modality_icon(m)} {m}" for m in modalities ) if modalities else "💫 Unknown" # Create model name with HTML link if URL exists model_display = f'{model}' if model_url else model # Create entry with numerical scores model_entry = { 'AI System': model_display, 'Modality': f"
{modality_badges}
", 'Overall Completion Rate': score_percentage } # Add selected category scores with emojis category_map = { '1. Bias, Stereotypes, and Representational Harms Evaluation': '⚖️ Bias and Fairness', '2. Cultural Values and Sensitive Content Evaluation': '🌍 Cultural Values', '3. Disparate Performance Evaluation': '📊 Disparate Performance', '4. Environmental Costs and Carbon Emissions Evaluation': '🌱 Environmental Impact', '5. Privacy and Data Protection Evaluation': '🔒 Privacy', '6. Financial Costs Evaluation': '💰 Financial Costs', '7. Data and Content Moderation Labor Evaluation': '👥 Labor Practices' } for full_cat_name, display_name in category_map.items(): if full_cat_name in selected_categories: score = score_by_category.get(full_cat_name, 0) model_entry[display_name] = score scores.append(model_entry) # Convert to DataFrame df = pd.DataFrame(scores) # Sort by Overall Completion Rate descending, putting N/A at the end df['_sort_value'] = df['Overall Completion Rate'].apply( lambda x: -float('inf') if x == "N/A" else float(x) ) df = df.sort_values('_sort_value', ascending=False) df = df.drop('_sort_value', axis=1) # Add rank column based on current sort df.insert(0, 'Rank', range(1, len(df) + 1)) # Get completion rate columns (Overall + category-specific) completion_rate_columns = ['Overall Completion Rate'] + [ display_name for full_cat_name, display_name in category_map.items() if full_cat_name in selected_categories ] # Format non-completion rate columns df['Rank'] = df['Rank'].astype(str) # Identify and format highest values for completion rate columns for col in completion_rate_columns: if col in df.columns: # Filter out N/A values to find the maximum numerical value numeric_values = df[df[col] != "N/A"][col] if not numeric_values.empty: max_value = numeric_values.max() df[col] = df.apply( lambda row: "N/A" if row[col] == "N/A" else f"**{row[col]:.1f}%**" if row[col] == max_value else f"{row[col]:.1f}%", axis=1 ) else: df[col] = df[col].apply(lambda x: "N/A") return df first_model = next(iter(models.values())) category_choices = list(first_model['scores'].keys()) with gr.Column(visible=True) as leaderboard_tab: leaderboard_output = gr.DataFrame( value=create_leaderboard(category_choices), interactive=False, wrap=True, datatype=["markdown", "markdown", "markdown"] + ["markdown"] * (len(category_choices)+1) # Support markdown in all columns ) def hex_to_rgba(hex_color, alpha): """Convert hex color to rgba string with given alpha value.""" hex_color = hex_color.lstrip('#') r = int(hex_color[:2], 16) g = int(hex_color[2:4], 16) b = int(hex_color[4:], 16) return f'rgba({r},{g},{b},{alpha})' def create_category_chart(selected_systems, selected_categories): if not selected_systems: fig = go.Figure() fig.add_annotation( text="Please select at least one AI system for comparison", xref="paper", yref="paper", x=0.5, y=0.5, showarrow=False ) return fig selected_categories = sort_categories(selected_categories) BASE_SCORE = 5 # Prepare all data first all_data = [] for system_name in selected_systems: system_data = [] for category in selected_categories: if category in models[system_name]['scores']: completed = 0 total = 0 category_name = category.split('.')[1].strip() all_na = True for section in models[system_name]['scores'][category].values(): if section['status'] != 'N/A': all_na = False questions = section.get('questions', {}) completed += sum(1 for q in questions.values() if q) total += len(questions) if all_na: score = BASE_SCORE display_score = 0 status = 'N/A' elif total > 0: raw_score = (completed / total) * 100 score = BASE_SCORE + (90 * raw_score / 100) display_score = raw_score status = 'Active' else: score = BASE_SCORE display_score = 0 status = 'Active' system_data.append({ 'AI System': system_name, 'Category': category_name, 'Score': score, 'Display Score': display_score, 'Status': status, 'Original Score': f"{display_score:.1f}%", 'Completed': completed, 'Total': total }) if system_data: # Add first point again to close the shape system_data.append(system_data[0].copy()) all_data.extend(system_data) df = pd.DataFrame(all_data) if df.empty: fig = go.Figure() fig.add_annotation( text="No data available for the selected AI systems and categories", xref="paper", yref="paper", x=0.5, y=0.5, showarrow=False ) return fig fig = go.Figure() # Define colors colors = [ '#FF4B4B', '#4B7BFF', '#4BFF4B', '#FFD700', '#FF4BFF', '#4BFFFF', '#FF884B', '#884BFF', '#4BFF88', '#FFFF4B' ] # Calculate average scores for sorting system_scores = { system: df[df['AI System'] == system]['Score'].mean() for system in selected_systems } sorted_systems = sorted(selected_systems, key=lambda x: system_scores[x], reverse=True) # Plot each system for idx, system_name in enumerate(sorted_systems): system_df = df[df['AI System'] == system_name] # Get color for this system base_color = colors[idx % len(colors)] line_color = hex_to_rgba(base_color, 0.9) fill_color = hex_to_rgba(base_color, 0.15) hover_color = hex_to_rgba(base_color, 1.0) # First, add the complete shape with all points (including N/A) fig.add_trace(go.Scatterpolar( r=system_df['Score'].tolist(), theta=system_df['Category'].tolist(), name=system_name, fill='toself', line=dict(color=line_color), fillcolor=fill_color, hoverinfo='skip', # Disable hover for the shape trace showlegend=True )) # Then add separate trace for hover information on non-N/A points non_na_df = system_df[system_df['Status'] != 'N/A'] if not non_na_df.empty: fig.add_trace(go.Scatterpolar( r=non_na_df['Score'].tolist(), theta=non_na_df['Category'].tolist(), mode='markers', marker=dict(size=1, color='rgba(0,0,0,0)'), # Nearly invisible markers customdata=list(zip( non_na_df['Original Score'], non_na_df['Status'], non_na_df['Completed'], non_na_df['Total'] )), hovertemplate=( f"" + "%{theta}
" + f"AI System: {system_name}
" + "Score: %{customdata[0]}
" + "Status: %{customdata[1]}
" + "Evaluations completed: %{customdata[2]}/%{customdata[3]}" + "" + ""), showlegend=False )) # Finally add N/A markers na_df = system_df[system_df['Status'] == 'N/A'] if not na_df.empty: fig.add_trace(go.Scatterpolar( r=na_df['Score'].tolist(), theta=na_df['Category'].tolist(), mode='markers+lines', line=dict(color='rgba(128, 128, 128, 0.3)', dash='dot'), marker=dict(color='rgba(128, 128, 128, 0.3)', size=8), customdata=list(zip( na_df['Original Score'], na_df['Status'], na_df['Completed'], na_df['Total'] )), hovertemplate="%{theta}
" + f"AI System: {system_name}
" + "Status: N/A
" + "Evaluations completed: %{customdata[2]}/%{customdata[3]}
" + "", showlegend=False )) # Update layout fig.update_layout( polar=dict( radialaxis=dict( visible=True, range=[0, 100], ticksuffix='%', showline=True, linewidth=1, gridwidth=1, gridcolor='rgba(0,0,0,0.1)', ticktext=[f'{i}%' for i in range(0, 101, 20)], tickvals=list(range(0, 101, 20)) ), angularaxis=dict( gridcolor='rgba(0,0,0,0.1)', linecolor='rgba(0,0,0,0.1)', ) ), showlegend=True, title=dict( text='Category Completion Rates by AI System', x=0.5, xanchor='center' ), legend=dict( yanchor="top", y=1.2, xanchor="left", x=1.1 ), margin=dict(t=100, b=100, l=100, r=100) ) return fig def update_detailed_scorecard(model, selected_categories): if not model: return [ gr.update(value="Please select a model to view details.", visible=True), gr.update(visible=False), gr.update(visible=False) ] selected_categories = sort_categories(selected_categories) metadata_html = create_metadata_card(models[model]['metadata']) overall_summary_html = create_overall_summary(models[model], selected_categories) # Combine metadata and overall summary combined_header = metadata_html + overall_summary_html total_yes = 0 total_no = 0 total_na = 0 has_non_na = False # Create category cards all_cards_content = "
" for category_name in selected_categories: if category_name in models[model]['scores']: category_data = models[model]['scores'][category_name] card_content = f"
{category_name}
" # Add category-specific summary at the top of each card card_content += create_category_summary(category_data) # Sort sections within each category sorted_sections = sorted(category_data.items(), key=lambda x: float(re.match(r'^(\d+\.?\d*)', x[0]).group(1))) category_yes = 0 category_no = 0 category_na = 0 for section, details in sorted_sections: status = details['status'] if status != 'N/A': has_non_na = True sources = details.get('sources', []) questions = details.get('questions', {}) section_class = "section-na" if status == "N/A" else "section-active" status_class = status.lower() status_icon = "●" if status == "Yes" else "○" if status == "N/A" else "×" card_content += f"
" card_content += f"

{section}

" card_content += f"{status_icon} {status}
" if sources: card_content += "
" for source in sources: icon = source.get("type", "") detail = source.get("detail", "") name = source.get("name", detail) card_content += f"
{icon} " if detail.startswith("http"): card_content += f"{name}" else: card_content += name card_content += "
" card_content += "
" if questions: yes_count = sum(1 for v in questions.values() if v) total_count = len(questions) card_content += "
" if status == "N/A": card_content += f"View {total_count} N/A items" else: card_content += f"View details ({yes_count}/{total_count} completed)" card_content += "
" for question, is_checked in questions.items(): if status == "N/A": style_class = "na" icon = "○" category_na += 1 total_na += 1 else: if is_checked: style_class = "checked" icon = "✓" category_yes += 1 total_yes += 1 else: style_class = "unchecked" icon = "✗" category_no += 1 total_no += 1 card_content += f"
{icon} {question}
" card_content += "
" card_content += "
" if category_yes + category_no > 0: category_score = category_yes / (category_yes + category_no) * 100 card_content += f"
Completion Score Breakdown: {category_score:.2f}% Yes: {category_yes}, No: {category_no}, N/A: {category_na}
" elif category_na > 0: card_content += f"
Completion Score Breakdown: N/A (All {category_na} items not applicable)
" card_content += "
" all_cards_content += card_content all_cards_content += "
" # Create total score if not has_non_na: total_score_md = "
No applicable scores (all items N/A)
" elif total_yes + total_no > 0: total_score = total_yes / (total_yes + total_no) * 100 total_score_md = f"
Total Score: {total_score:.2f}% (Yes: {total_yes}, No: {total_no}, N/A: {total_na})
" else: total_score_md = "
No applicable scores (all items N/A)
" return [ gr.update(value=combined_header, visible=True), gr.update(value=all_cards_content, visible=True), gr.update(value=total_score_md, visible=True) ] first_model = next(iter(models.values())) category_choices = list(first_model['scores'].keys()) with gr.Blocks(css=css) as demo: gr.Markdown("# AI System Social Impact Dashboard") initial_df = create_leaderboard(category_choices) with gr.Row(): tab_selection = gr.Radio(["Leaderboard", "Category Analysis", "Detailed Scorecard"], label="Select Tab", value="Leaderboard") with gr.Row(): model_chooser = gr.Dropdown(choices=[""] + list(models.keys()), label="Select AI System for Details", value="", interactive=True, visible=False) model_multi_chooser = gr.Dropdown(choices=list(models.keys()), label="Select AI Systems for Comparison", value=[], multiselect=True, interactive=True, visible=False, info="Select one or more AI Systems") # Category filter now visible for all tabs category_filter = gr.CheckboxGroup(choices=category_choices, label="Filter Categories", value=category_choices) with gr.Column(visible=True) as leaderboard_tab: leaderboard_output = gr.DataFrame( value=initial_df, interactive=False, wrap=True, datatype=["markdown", "markdown", "markdown"] + ["markdown"] * (len(category_choices)+1) # Support markdown in all columns ) with gr.Column(visible=False) as category_analysis_tab: # Initialize with empty plot initial_plot = create_category_chart([], category_choices) category_chart = gr.Plot(value=initial_plot) with gr.Column(visible=False) as detailed_scorecard_tab: model_metadata = gr.HTML() all_category_cards = gr.HTML() total_score = gr.Markdown() def update_dashboard(tab, selected_models, selected_model, selected_categories): # Default visibility states component_states = { "leaderboard": False, "category_chart": False, "detailed_scorecard": False, "model_chooser": False, "model_multi_chooser": False } # Initialize outputs with None outputs = { "leaderboard": None, "category_chart": None, "model_metadata": None, "category_cards": None, "total_score": None } # Update visibility based on selected tab if tab == "Leaderboard": component_states["leaderboard"] = True outputs["leaderboard"] = create_leaderboard(selected_categories) elif tab == "Category Analysis": component_states["category_chart"] = True component_states["model_multi_chooser"] = True if selected_models: # Only update chart if models are selected outputs["category_chart"] = create_category_chart(selected_models, selected_categories) elif tab == "Detailed Scorecard": component_states["detailed_scorecard"] = True component_states["model_chooser"] = True if selected_model: scorecard_updates = update_detailed_scorecard(selected_model, selected_categories) outputs["model_metadata"] = scorecard_updates[0] outputs["category_cards"] = scorecard_updates[1] outputs["total_score"] = scorecard_updates[2] # Return updates in the correct order return [ gr.update(visible=component_states["leaderboard"]), gr.update(visible=component_states["category_chart"]), gr.update(visible=component_states["detailed_scorecard"]), gr.update(visible=component_states["model_chooser"]), gr.update(visible=component_states["model_multi_chooser"]), outputs["leaderboard"] if outputs["leaderboard"] is not None else gr.update(), outputs["category_chart"] if outputs["category_chart"] is not None else gr.update(), outputs["model_metadata"] if outputs["model_metadata"] is not None else gr.update(), outputs["category_cards"] if outputs["category_cards"] is not None else gr.update(), outputs["total_score"] if outputs["total_score"] is not None else gr.update() ] # Set up event handlers for component in [tab_selection, model_chooser, model_multi_chooser, category_filter]: component.change( fn=update_dashboard, inputs=[tab_selection, model_multi_chooser, model_chooser, category_filter], outputs=[leaderboard_tab, category_analysis_tab, detailed_scorecard_tab, model_chooser, model_multi_chooser, leaderboard_output, category_chart, model_metadata, all_category_cards, total_score] ) # Launch the app if __name__ == "__main__": demo.launch(ssr_mode=False)