Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| from huggingface_hub import list_models | |
| import plotly.express as px | |
| tasks = ['asr.csv', 'object_detection.csv', 'text_classification.csv', 'image_captioning.csv', | |
| 'question_answering.csv', 'text_generation.csv', 'image_classification.csv', | |
| 'sentence_similarity.csv', 'image_generation.csv', 'summarization.csv'] | |
| def get_plots(task): | |
| #TO DO : hover text with energy efficiency number, parameters | |
| task_df= pd.read_csv('data/energy/'+task) | |
| params_df = pd.read_csv('data/params/'+task) | |
| params_df= params_df.rename(columns={"Link": "model"}) | |
| all_df = pd.merge(task_df, params_df, on='model') | |
| all_df['Total GPU Energy (Wh)'] = all_df['total_gpu_energy']*1000 | |
| all_df = all_df.sort_values(by=['Total GPU Energy (Wh)']) | |
| all_df['parameters'] = all_df['parameters'].apply(format_params) | |
| all_df['energy_star'] = pd.cut(all_df['Total GPU Energy (Wh)'], 3, labels=["⭐⭐⭐", "⭐⭐", "⭐"]) | |
| fig = px.scatter(all_df, x="model", y='Total GPU Energy (Wh)', custom_data=['parameters'], height= 500, width= 800, color = 'energy_star', color_discrete_map={"⭐": 'red', "⭐⭐": "yellow", "⭐⭐⭐": "green"}) | |
| fig.update_traces( | |
| hovertemplate="<br>".join([ | |
| "Total Energy: %{y}", | |
| "Parameters: %{customdata[0]}"]) | |
| ) | |
| return fig | |
| def get_all_plots(): | |
| for task in tasks: | |
| task_df= pd.read_csv('data/energy/'+task) | |
| params_df = pd.read_csv('data/params/'+task) | |
| params_df= params_df.rename(columns={"Link": "model"}) | |
| tasks_df = pd.merge(task_df, params_df, on='model') | |
| all_df = pd.DataFrame(columns = tasks_df.columns) | |
| all_df = all_df.append(tasks_df) | |
| all_df['Total GPU Energy (Wh)'] = all_df['total_gpu_energy']*1000 | |
| all_df = all_df.sort_values(by=['Total GPU Energy (Wh)']) | |
| all_df['parameters'] = all_df['parameters'].apply(format_params) | |
| all_df['energy_star'] = pd.cut(all_df['Total GPU Energy (Wh)'], 3, labels=["⭐⭐⭐", "⭐⭐", "⭐"]) | |
| fig = px.scatter(all_df, x="model", y='Total GPU Energy (Wh)', custom_data=['parameters'], height= 500, width= 800, color = 'energy_star', color_discrete_map={"⭐": 'red', "⭐⭐": "yellow", "⭐⭐⭐": "green"}) | |
| fig.update_traces( | |
| hovertemplate="<br>".join([ | |
| "Total Energy: %{y}", | |
| "Parameters: %{customdata[0]}"]) | |
| ) | |
| return fig | |
| def make_link(mname): | |
| link = "["+ str(mname).split('/')[1] +'](https://huggingface.co/'+str(mname)+")" | |
| return link | |
| def get_model_names(task): | |
| task_df= pd.read_csv('data/params/'+task) | |
| energy_df= pd.read_csv('data/energy/'+task) | |
| task_df= task_df.rename(columns={"Link": "model"}) | |
| all_df = pd.merge(task_df, energy_df, on='model') | |
| all_df=all_df.drop_duplicates(subset=['model']) | |
| all_df['Parameters'] = all_df['parameters'].apply(format_params) | |
| all_df['Model'] = all_df['model'].apply(make_link) | |
| all_df['Total GPU Energy (Wh)'] = all_df['total_gpu_energy']*1000 | |
| all_df['Total GPU Energy (Wh)'] = all_df['Total GPU Energy (Wh)'].round(2) | |
| all_df['Rating'] = pd.cut(all_df['Total GPU Energy (Wh)'], 3, labels=["⭐⭐⭐", "⭐⭐", "⭐"]) | |
| model_names= model_names.sort_values('Total GPU Energy (Wh)') | |
| model_names = all_df[['Model','Rating','Total GPU Energy (Wh)', 'Parameters']] | |
| return model_names | |
| def get_all_model_names(): | |
| #TODO: add link to results in model card of each model | |
| for task in tasks: | |
| task_df= pd.read_csv('data/params/'+task) | |
| energy_df= pd.read_csv('data/energy/'+task) | |
| task_df= task_df.rename(columns={"Link": "model"}) | |
| tasks_df = pd.merge(task_df, energy_df, on='model') | |
| all_df = pd.DataFrame(columns = tasks_df.columns) | |
| all_df = all_df.append(tasks_df) | |
| all_df=all_df.drop_duplicates(subset=['model']) | |
| all_df['Parameters'] = all_df['parameters'].apply(format_params) | |
| all_df['Model'] = all_df['model'].apply(make_link) | |
| all_df['Total GPU Energy (Wh)'] = all_df['total_gpu_energy']*1000 | |
| all_df['Total GPU Energy (Wh)'] = all_df['Total GPU Energy (Wh)'].round(2) | |
| all_df['Rating'] = pd.cut(all_df['Total GPU Energy (Wh)'], 3, labels=["⭐⭐⭐", "⭐⭐", "⭐"]) | |
| model_names= model_names.sort_values('Total GPU Energy (Wh)') | |
| model_names = all_df[['Model','Rating','Total GPU Energy (Wh)', 'Parameters']] | |
| return model_names | |
| def format_params(num): | |
| if num > 1000000000: | |
| if not num % 1000000000: | |
| return f'{num // 1000000000}B' | |
| return f'{round(num / 1000000000, 1)}B' | |
| return f'{num // 1000000}M' | |
| demo = gr.Blocks() | |
| with demo: | |
| gr.Markdown( | |
| """# Energy Star Leaderboard - v.0 (2024) 🌎 💻 🌟 | |
| ### Welcome to the leaderboard for the [AI Energy Star Project!](https://huggingface.co/EnergyStarAI) | |
| Click through the tasks below to see how different models measure up in terms of energy efficiency""" | |
| ) | |
| gr.Markdown( | |
| """Test your own models via the [submission portal (TODO)].""" | |
| ) | |
| with gr.Tabs(): | |
| with gr.TabItem("Text Generation 💬"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('text_generation.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('text_generation.csv'), datatype="markdown") | |
| with gr.TabItem("Image Generation 📷"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('image_generation.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('image_generation.csv'), datatype="markdown") | |
| with gr.TabItem("Text Classification 🎭"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('text_classification.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('text_classification.csv'), datatype="markdown") | |
| with gr.TabItem("Image Classification 🖼️"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('image_classification.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('image_classification.csv'), datatype="markdown") | |
| with gr.TabItem("Image Captioning 📝"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('question_answering.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('question_answering.csv'), datatype="markdown") | |
| with gr.TabItem("Summarization 📃"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('summarization.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('summarization.csv'), datatype="markdown") | |
| with gr.TabItem("Automatic Speech Recognition 💬 "): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('asr.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('asr.csv'), datatype="markdown") | |
| with gr.TabItem("Object Detection 🚘"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('object_detection.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('object_detection.csv'), datatype="markdown") | |
| with gr.TabItem("Sentence Similarity 📚"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('sentence_similarity.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('sentence_similarity.csv'), datatype="markdown") | |
| with gr.TabItem("Extractive QA ❔"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_plots('question_answering.csv')) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_model_names('question_answering.csv'), datatype="markdown") | |
| with gr.TabItem("Overall"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| plot = gr.Plot(get_all_plots) | |
| with gr.Column(): | |
| table = gr.Dataframe(get_all_model_names) | |
| with gr.Accordion("Methodology", open = False): | |
| gr.Markdown( | |
| """For each of the ten tasks above, we created a custom dataset with 1,000 entries (see all of the datasets on our [org Hub page](https://huggingface.co/EnergyStarAI)). | |
| We then tested each of the models from the leaderboard on the appropriate task on Nvidia A100 GPUs, measuring the energy consumed using [Code Carbon](https://mlco2.github.io/codecarbon/), an open-source Python package for tracking the environmental impacts of code. | |
| We developed and used a [Docker container](https://github.com/huggingface/EnergyStarAI/) to maximize the reproducibility of results, and to enable members of the community to benchmark internal models. | |
| Reach out to us if you want to collaborate! | |
| """) | |
| gr.Markdown( | |
| """Last updated: September 20th, 2024 by [Sasha Luccioni](https://huggingface.co/sasha)""") | |
| demo.launch() | |