evijit HF staff commited on
Commit
47a7ca5
1 Parent(s): a226d69

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +263 -30
app.py CHANGED
@@ -6,6 +6,7 @@ from typing import List, Dict, Tuple, Union
6
  import json
7
  import os
8
  from collections import OrderedDict
 
9
 
10
  @dataclass
11
  class ScorecardCategory:
@@ -13,17 +14,82 @@ class ScorecardCategory:
13
  questions: List[Dict[str, Union[str, List[str]]]]
14
  scores: Dict[str, int] = field(default_factory=dict)
15
 
16
- def load_scorecard_templates(directory):
17
- templates = []
18
- for filename in os.listdir(directory):
19
- if filename.endswith('.json'):
20
- with open(os.path.join(directory, filename), 'r') as file:
21
- data = json.load(file)
22
- templates.append(ScorecardCategory(
23
- name=data['name'],
24
- questions=data['questions']
25
- ))
26
- return templates
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
  def load_models_from_json(directory):
29
  models = {}
@@ -37,7 +103,7 @@ def load_models_from_json(directory):
37
  return OrderedDict(sorted(models.items(), key=lambda x: x[0].lower()))
38
 
39
  # Load templates and models
40
- scorecard_template = load_scorecard_templates('scorecard_templates')
41
  models = load_models_from_json('model_data')
42
 
43
  def create_source_html(sources):
@@ -92,6 +158,9 @@ def create_category_chart(selected_models, selected_categories):
92
  if not selected_models:
93
  return px.bar(title='Please select at least one model for comparison')
94
 
 
 
 
95
  data = []
96
  for model in selected_models:
97
  for category in selected_categories:
@@ -129,10 +198,18 @@ def update_detailed_scorecard(model, selected_categories):
129
  gr.update(visible=False),
130
  gr.update(visible=False)
131
  ]
 
 
 
 
 
 
 
132
 
133
- metadata_md = f"## Model Metadata for {model}\n\n"
134
- for key, value in models[model]['metadata'].items():
135
- metadata_md += f"**{key}:** {value}\n\n"
 
136
 
137
  total_yes = 0
138
  total_no = 0
@@ -144,21 +221,27 @@ def update_detailed_scorecard(model, selected_categories):
144
  category_data = models[model]['scores'][category_name]
145
  card_content = f"<div class='card'><div class='card-title'>{category_name}</div>"
146
 
 
 
 
 
147
  category_yes = 0
148
  category_no = 0
149
  category_na = 0
150
 
151
- for section, details in category_data.items():
152
  status = details['status']
153
  sources = details.get('sources', [])
154
  questions = details.get('questions', {})
155
 
156
- # Determine section class based on status
157
  section_class = "section-na" if status == "N/A" else "section-active"
 
 
 
158
  card_content += f"<div class='section {section_class}'>"
159
- card_content += f"<h3>{section}</h3>"
 
160
 
161
- # Add sources if they exist
162
  if sources:
163
  card_content += "<div class='sources-list'>"
164
  for source in sources:
@@ -174,13 +257,21 @@ def update_detailed_scorecard(model, selected_categories):
174
  card_content += "</div>"
175
  card_content += "</div>"
176
 
177
- # Process questions
178
  if questions:
 
 
 
 
 
 
 
 
 
179
  card_content += "<div class='questions'>"
180
  for question, is_checked in questions.items():
181
  if status == "N/A":
182
  style_class = "na"
183
- icon = "○" # Circle for N/A items
184
  category_na += 1
185
  total_na += 1
186
  else:
@@ -196,23 +287,21 @@ def update_detailed_scorecard(model, selected_categories):
196
  total_no += 1
197
 
198
  card_content += f"<div class='question-item {style_class}'>{icon} {question}</div>"
199
- card_content += "</div>"
200
 
201
- card_content += "</div>" # Close section div
202
 
203
- # Calculate category score (excluding N/A items)
204
  if category_yes + category_no > 0:
205
  category_score = category_yes / (category_yes + category_no) * 100
206
  card_content += f"<div class='category-score'>Category Score: {category_score:.2f}% (Yes: {category_yes}, No: {category_no}, N/A: {category_na})</div>"
207
  elif category_na > 0:
208
  card_content += f"<div class='category-score'>Category Score: N/A (All {category_na} items not applicable)</div>"
209
 
210
- card_content += "</div>" # Close card div
211
  all_cards_content += card_content
212
 
213
  all_cards_content += "</div>"
214
 
215
- # Calculate total score (excluding N/A items)
216
  if total_yes + total_no > 0:
217
  total_score = total_yes / (total_yes + total_no) * 100
218
  total_score_md = f"<div class='total-score'>Total Score: {total_score:.2f}% (Yes: {total_yes}, No: {total_no}, N/A: {total_na})</div>"
@@ -220,7 +309,7 @@ def update_detailed_scorecard(model, selected_categories):
220
  total_score_md = "<div class='total-score'>No applicable scores (all items N/A)</div>"
221
 
222
  return [
223
- gr.update(value=metadata_md, visible=True),
224
  gr.update(value=all_cards_content, visible=True),
225
  gr.update(value=total_score_md, visible=True)
226
  ]
@@ -365,8 +454,152 @@ css = """
365
  background-color: #2d2d2d;
366
  color: #999;
367
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
368
  """
369
 
 
 
 
370
  with gr.Blocks(css=css) as demo:
371
  gr.Markdown("# AI Model Social Impact Scorecard Dashboard")
372
 
@@ -382,9 +615,9 @@ with gr.Blocks(css=css) as demo:
382
  model_multi_chooser = gr.Dropdown(choices=list(models.keys()),
383
  label="Select Models for Comparison",
384
  multiselect=True, interactive=True, visible=False)
385
- category_filter = gr.CheckboxGroup(choices=[cat.name for cat in scorecard_template],
386
  label="Filter Categories",
387
- value=[cat.name for cat in scorecard_template],
388
  visible=False)
389
 
390
  with gr.Column(visible=True) as leaderboard_tab:
@@ -394,7 +627,7 @@ with gr.Blocks(css=css) as demo:
394
  category_chart = gr.Plot()
395
 
396
  with gr.Column(visible=False) as detailed_scorecard_tab:
397
- model_metadata = gr.Markdown()
398
  all_category_cards = gr.HTML()
399
  total_score = gr.Markdown()
400
 
 
6
  import json
7
  import os
8
  from collections import OrderedDict
9
+ import re
10
 
11
  @dataclass
12
  class ScorecardCategory:
 
14
  questions: List[Dict[str, Union[str, List[str]]]]
15
  scores: Dict[str, int] = field(default_factory=dict)
16
 
17
+
18
+ def extract_category_number(category_name: str) -> int:
19
+ """Extract the category number from the category name."""
20
+ match = re.match(r'^(\d+)\.?\s*.*$', category_name)
21
+ return int(match.group(1)) if match else float('inf')
22
+
23
+ def sort_categories(categories):
24
+ """Sort categories by their numeric prefix."""
25
+ return sorted(categories, key=extract_category_number)
26
+
27
+
28
+ # def load_scorecard_templates(directory):
29
+ # templates = []
30
+ # for filename in os.listdir(directory):
31
+ # if filename.endswith('.json'):
32
+ # with open(os.path.join(directory, filename), 'r') as file:
33
+ # data = json.load(file)
34
+ # templates.append(ScorecardCategory(
35
+ # name=data['name'],
36
+ # questions=data['questions']
37
+ # ))
38
+ # return templates
39
+
40
+ def get_modality_icon(modality):
41
+ """Return an emoji icon for each modality type."""
42
+ icons = {
43
+ "Text-to-Text": "📝", # Memo icon for text-to-text
44
+ "Text-to-Image": "🎨", # Artist palette for text-to-image
45
+ "Image-to-Text": "🔍", # Magnifying glass for image-to-text
46
+ "Image-to-Image": "🖼️", # Frame for image-to-image
47
+ "Audio": "🎵", # Musical note for audio
48
+ "Video": "🎬", # Clapper board for video
49
+ "Multimodal": "🔄" # Cycle arrows for multimodal
50
+ }
51
+ return icons.get(modality, "💫") # Default icon if modality not found
52
+
53
+ def create_metadata_card(metadata):
54
+ """Create a formatted HTML card for metadata."""
55
+ html = "<div class='card metadata-card'>"
56
+ html += "<div class='card-title'>Model Information</div>"
57
+ html += "<div class='metadata-content'>"
58
+
59
+ # Handle special formatting for modalities
60
+ modalities = metadata.get("Modalities", [])
61
+ formatted_modalities = ""
62
+ if modalities:
63
+ formatted_modalities = " ".join(
64
+ f"<span class='modality-badge'>{get_modality_icon(m)} {m}</span>"
65
+ for m in modalities
66
+ )
67
+
68
+ # Order of metadata display (customize as needed)
69
+ display_order = ["Name", "Provider", "Type", "URL"]
70
+
71
+ # Display ordered metadata first
72
+ for key in display_order:
73
+ if key in metadata:
74
+ value = metadata[key]
75
+ if key == "URL":
76
+ html += f"<div class='metadata-row'><span class='metadata-label'>{key}:</span> "
77
+ html += f"<a href='{value}' target='_blank' class='metadata-link'>{value}</a></div>"
78
+ else:
79
+ html += f"<div class='metadata-row'><span class='metadata-label'>{key}:</span> <span class='metadata-value'>{value}</span></div>"
80
+
81
+ # Add modalities if present
82
+ if formatted_modalities:
83
+ html += f"<div class='metadata-row'><span class='metadata-label'>Modalities:</span> <div class='modality-container'>{formatted_modalities}</div></div>"
84
+
85
+ # Add any remaining metadata not in display_order
86
+ for key, value in metadata.items():
87
+ if key not in display_order and key != "Modalities":
88
+ html += f"<div class='metadata-row'><span class='metadata-label'>{key}:</span> <span class='metadata-value'>{value}</span></div>"
89
+
90
+ html += "</div></div>"
91
+ return html
92
+
93
 
94
  def load_models_from_json(directory):
95
  models = {}
 
103
  return OrderedDict(sorted(models.items(), key=lambda x: x[0].lower()))
104
 
105
  # Load templates and models
106
+ # scorecard_template = load_scorecard_templates('scorecard_templates')
107
  models = load_models_from_json('model_data')
108
 
109
  def create_source_html(sources):
 
158
  if not selected_models:
159
  return px.bar(title='Please select at least one model for comparison')
160
 
161
+ # Sort categories before processing
162
+ selected_categories = sort_categories(selected_categories)
163
+
164
  data = []
165
  for model in selected_models:
166
  for category in selected_categories:
 
198
  gr.update(visible=False),
199
  gr.update(visible=False)
200
  ]
201
+
202
+ print("Selected categories:", selected_categories)
203
+ print("Available categories in model:", list(models[model]['scores'].keys()))
204
+
205
+ # Sort categories before processing
206
+ selected_categories = sort_categories(selected_categories)
207
+ metadata_html = create_metadata_card(models[model]['metadata'])
208
 
209
+
210
+ # metadata_md = f"## Model Metadata for {model}\n\n"
211
+ # for key, value in models[model]['metadata'].items():
212
+ # metadata_md += f"**{key}:** {value}\n\n"
213
 
214
  total_yes = 0
215
  total_no = 0
 
221
  category_data = models[model]['scores'][category_name]
222
  card_content = f"<div class='card'><div class='card-title'>{category_name}</div>"
223
 
224
+ # Sort sections within each category
225
+ sorted_sections = sorted(category_data.items(),
226
+ key=lambda x: float(re.match(r'^(\d+\.?\d*)', x[0]).group(1)))
227
+
228
  category_yes = 0
229
  category_no = 0
230
  category_na = 0
231
 
232
+ for section, details in sorted_sections:
233
  status = details['status']
234
  sources = details.get('sources', [])
235
  questions = details.get('questions', {})
236
 
 
237
  section_class = "section-na" if status == "N/A" else "section-active"
238
+ status_class = status.lower()
239
+ status_icon = "●" if status == "Yes" else "○" if status == "N/A" else "×"
240
+
241
  card_content += f"<div class='section {section_class}'>"
242
+ card_content += f"<div class='section-header'><h3>{section}</h3>"
243
+ card_content += f"<span class='status-badge {status_class}'>{status_icon} {status}</span></div>"
244
 
 
245
  if sources:
246
  card_content += "<div class='sources-list'>"
247
  for source in sources:
 
257
  card_content += "</div>"
258
  card_content += "</div>"
259
 
 
260
  if questions:
261
+ yes_count = sum(1 for v in questions.values() if v)
262
+ total_count = len(questions)
263
+
264
+ card_content += "<details class='question-accordion'>"
265
+ if status == "N/A":
266
+ card_content += f"<summary>View {total_count} N/A items</summary>"
267
+ else:
268
+ card_content += f"<summary>View details ({yes_count}/{total_count} completed)</summary>"
269
+
270
  card_content += "<div class='questions'>"
271
  for question, is_checked in questions.items():
272
  if status == "N/A":
273
  style_class = "na"
274
+ icon = "○"
275
  category_na += 1
276
  total_na += 1
277
  else:
 
287
  total_no += 1
288
 
289
  card_content += f"<div class='question-item {style_class}'>{icon} {question}</div>"
290
+ card_content += "</div></details>"
291
 
292
+ card_content += "</div>"
293
 
 
294
  if category_yes + category_no > 0:
295
  category_score = category_yes / (category_yes + category_no) * 100
296
  card_content += f"<div class='category-score'>Category Score: {category_score:.2f}% (Yes: {category_yes}, No: {category_no}, N/A: {category_na})</div>"
297
  elif category_na > 0:
298
  card_content += f"<div class='category-score'>Category Score: N/A (All {category_na} items not applicable)</div>"
299
 
300
+ card_content += "</div>"
301
  all_cards_content += card_content
302
 
303
  all_cards_content += "</div>"
304
 
 
305
  if total_yes + total_no > 0:
306
  total_score = total_yes / (total_yes + total_no) * 100
307
  total_score_md = f"<div class='total-score'>Total Score: {total_score:.2f}% (Yes: {total_yes}, No: {total_no}, N/A: {total_na})</div>"
 
309
  total_score_md = "<div class='total-score'>No applicable scores (all items N/A)</div>"
310
 
311
  return [
312
+ gr.update(value=metadata_html, visible=True),
313
  gr.update(value=all_cards_content, visible=True),
314
  gr.update(value=total_score_md, visible=True)
315
  ]
 
454
  background-color: #2d2d2d;
455
  color: #999;
456
  }
457
+
458
+ .section-header {
459
+ display: flex;
460
+ justify-content: space-between;
461
+ align-items: center;
462
+ margin-bottom: 10px;
463
+ }
464
+
465
+ .status-badge {
466
+ font-size: 0.9em;
467
+ padding: 4px 8px;
468
+ border-radius: 12px;
469
+ font-weight: 500;
470
+ }
471
+
472
+ .status-badge.yes {
473
+ background-color: #e6ffe6;
474
+ color: #006600;
475
+ }
476
+
477
+ .status-badge.no {
478
+ background-color: #ffe6e6;
479
+ color: #990000;
480
+ }
481
+
482
+ .status-badge.n\/a {
483
+ background-color: #f0f0f0;
484
+ color: #666666;
485
+ }
486
+
487
+ .question-accordion {
488
+ margin-top: 10px;
489
+ }
490
+
491
+ .question-accordion summary {
492
+ cursor: pointer;
493
+ padding: 8px;
494
+ background-color: #f8f9fa;
495
+ border-radius: 4px;
496
+ margin-bottom: 10px;
497
+ font-weight: 500;
498
+ }
499
+
500
+ .question-accordion summary:hover {
501
+ background-color: #e9ecef;
502
+ }
503
+
504
+ .dark .status-badge.yes {
505
+ background-color: #1a3a1a;
506
+ color: #90EE90;
507
+ }
508
+
509
+ .dark .status-badge.no {
510
+ background-color: #3a1a1a;
511
+ color: #FFB6B6;
512
+ }
513
+
514
+ .dark .status-badge.n\/a {
515
+ background-color: #2d2d2d;
516
+ color: #999999;
517
+ }
518
+
519
+ .dark .question-accordion summary {
520
+ background-color: #2a2a2a;
521
+ }
522
+
523
+ .dark .question-accordion summary:hover {
524
+ background-color: #333333;
525
+ }
526
+ .metadata-card {
527
+ margin-bottom: 30px;
528
+ width: 100% !important;
529
+ }
530
+
531
+ .metadata-content {
532
+ display: flex;
533
+ flex-direction: column;
534
+ gap: 12px;
535
+ }
536
+
537
+ .metadata-row {
538
+ display: flex;
539
+ align-items: flex-start;
540
+ gap: 10px;
541
+ line-height: 1.5;
542
+ }
543
+
544
+ .metadata-label {
545
+ font-weight: 600;
546
+ min-width: 100px;
547
+ color: #555;
548
+ }
549
+
550
+ .metadata-value {
551
+ color: #333;
552
+ }
553
+
554
+ .metadata-link {
555
+ color: #007bff;
556
+ text-decoration: none;
557
+ }
558
+
559
+ .metadata-link:hover {
560
+ text-decoration: underline;
561
+ }
562
+
563
+ .modality-container {
564
+ display: flex;
565
+ flex-wrap: wrap;
566
+ gap: 8px;
567
+ }
568
+
569
+ .modality-badge {
570
+ display: inline-flex;
571
+ align-items: center;
572
+ gap: 4px;
573
+ padding: 4px 10px;
574
+ background-color: #f0f7ff;
575
+ border: 1px solid #cce3ff;
576
+ border-radius: 15px;
577
+ font-size: 0.9em;
578
+ color: #0066cc;
579
+ }
580
+
581
+ .dark .metadata-label {
582
+ color: #aaa;
583
+ }
584
+
585
+ .dark .metadata-value {
586
+ color: #ddd;
587
+ }
588
+
589
+ .dark .metadata-link {
590
+ color: #66b3ff;
591
+ }
592
+
593
+ .dark .modality-badge {
594
+ background-color: #1a2733;
595
+ border-color: #2c3e50;
596
+ color: #99ccff;
597
+ }
598
  """
599
 
600
+ first_model = next(iter(models.values()))
601
+ category_choices = list(first_model['scores'].keys())
602
+
603
  with gr.Blocks(css=css) as demo:
604
  gr.Markdown("# AI Model Social Impact Scorecard Dashboard")
605
 
 
615
  model_multi_chooser = gr.Dropdown(choices=list(models.keys()),
616
  label="Select Models for Comparison",
617
  multiselect=True, interactive=True, visible=False)
618
+ category_filter = gr.CheckboxGroup(choices=category_choices,
619
  label="Filter Categories",
620
+ value=category_choices,
621
  visible=False)
622
 
623
  with gr.Column(visible=True) as leaderboard_tab:
 
627
  category_chart = gr.Plot()
628
 
629
  with gr.Column(visible=False) as detailed_scorecard_tab:
630
+ model_metadata = gr.HTML()
631
  all_category_cards = gr.HTML()
632
  total_score = gr.Markdown()
633