LLMCalc / app.py
carbonnnnn's picture
update #2
1580227
import pandas as pd
import gradio as gr
import os
from gradio_rangeslider import RangeSlider
import math
from utils.filter_utils import filter, filter_cols
# Main Leaderboard containing everything
text_leaderboard = pd.read_csv(os.path.join('src', 'main_df.csv'))
text_leaderboard = text_leaderboard.sort_values(by='Average Clemscore', ascending=False)
text_leaderboard = text_leaderboard.sort_values(by='Average Clemscore', ascending=False)
open_weight_df = text_leaderboard[text_leaderboard['Open Weight'] == True]
if not open_weight_df.empty: # Check if filtered df is non-empty
max_parameter_size = open_weight_df['Parameter Size (B)'].max()
# Short leaderboard containing fixed columns
short_leaderboard = filter_cols(text_leaderboard)
## Extract data
langs = []
licenses = []
ip_prices = []
op_prices = []
latencies = []
parameters = []
contexts = []
dates = []
for i in range(len(text_leaderboard)):
lang_splits = text_leaderboard.iloc[i]['Languages'].split(',')
lang_splits = [s.strip() for s in lang_splits]
langs += lang_splits
license_name = text_leaderboard.iloc[i]['License Name']
licenses.append(license_name)
ip_prices.append(text_leaderboard.iloc[i]['Input $/1M'])
op_prices.append(text_leaderboard.iloc[i]['Output $/1M'])
latencies.append(text_leaderboard.iloc[i]['Average Latency (s)'])
parameters.append(text_leaderboard.iloc[i]['Parameter Size (B)'])
contexts.append(text_leaderboard.iloc[i]['Context Size (k)'])
dates.append(text_leaderboard.iloc[i]['Release Date'])
langs = list(set(langs))
langs.sort()
licenses = list(set(licenses))
licenses.sort()
max_input_price = max(ip_prices)
max_output_price = max(op_prices)
max_latency = max(latencies)
min_parameters = min(parameters)
max_parameter = max_parameter_size
parameter_step = 1
min_context = min(contexts)
max_context = max(contexts)
context_step = 8
min_date = min(dates)
max_date = max(dates)
TITLE = """<h1 align="center" id="space-title"> LLM Calculator βš–οΈβš‘ πŸ“πŸ’°</h1>"""
CSS = """
#double-slider-1 {height: 100px}
#double-slider-2 {height: 100px}
#double-slider-3 {height: 100px}
#double-slider-4 {height: 100px}
"""
llm_calc_app = gr.Blocks(css=CSS)
with llm_calc_app:
gr.HTML(TITLE)
##################################################
with gr.Row():
#####################################
# First Column
####################################
## Language Select
with gr.Column():
with gr.Row():
lang_dropdown = gr.Dropdown(
choices=langs,
value=[],
multiselect=True,
label="Select Languages πŸ—£οΈ"
)
with gr.Row():
with gr.Column():
start_date = gr.DateTime(
value=min_date,
type="string",
label="Release Date Range πŸ“… - Start Date"
)
with gr.Column():
end_date = gr.DateTime(
value=max_date,
type="string",
label="End Date"
)
# Multiodality Select
with gr.Row():
multimodal_checkbox = gr.CheckboxGroup(
choices=['Image', 'Multi-Image', 'Audio', 'Video'],
value=[],
label="Select Additional Modalities πŸ“·πŸŽ§πŸŽ¬",
)
# Open/Commercial Selection
with gr.Row():
open_weight_checkbox = gr.CheckboxGroup(
choices=['Open', 'Commercial'],
value=['Open', 'Commercial'],
label="Filter by Model Type πŸ”“ πŸ’Ό",
)
# License selection
with gr.Row():
license_checkbox = gr.CheckboxGroup(
choices=licenses,
value=licenses,
label="License Type πŸ›‘οΈ",
)
#############################################################
# Second Column
#############################################################
with gr.Column():
####### LOG SLIDER 1 ###########
with gr.Row():
parameter_slider = RangeSlider(
minimum=0,
maximum=max_parameter,
label=f"Select Parameter Range πŸ” {int(min_parameters)}B - {int(max_parameter)}B+",
elem_id="double-slider-1",
step=parameter_step
)
########### LOG SLIDER 2 ################
with gr.Row():
context_slider = RangeSlider(
minimum=0,
maximum=max_context,
label="Select Context Range (k) πŸ“",
elem_id="double-slider-2",
step=context_step
)
############# PRICE SLIDER 1 ###############
with gr.Row():
input_pricing_slider = RangeSlider(
minimum=0,
maximum=max_input_price,
value=(0, max_input_price),
label="Select Price range πŸ’²/1M input tokens",
elem_id="double-slider-3"
)
############### PRICE SLIDER 2 ###############
with gr.Row():
output_pricing_slider = RangeSlider(
minimum=0,
maximum=max_output_price,
value=(0, max_output_price),
label="Select Price range πŸ’²/1M output tokens",
elem_id="double-slider-4"
)
with gr.Row():
"""
Main Leaderboard Row
"""
leaderboard_table = gr.Dataframe(
value=short_leaderboard,
elem_id="text-leaderboard-table",
interactive=False,
visible=True,
height=800,
datatype=['html', 'number', 'number', 'date', 'number', 'number', 'number', 'number', 'html']
)
dummy_leaderboard_table = gr.Dataframe(
value=text_leaderboard,
elem_id="dummy-leaderboard-table",
interactive=False,
visible=False
)
lang_dropdown.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
parameter_slider.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
input_pricing_slider.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
output_pricing_slider.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
multimodal_checkbox.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
open_weight_checkbox.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
context_slider.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
start_date.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
end_date.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
license_checkbox.change(
filter,
[dummy_leaderboard_table, lang_dropdown, parameter_slider,
input_pricing_slider, output_pricing_slider, multimodal_checkbox,
context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
[leaderboard_table],
queue=True
)
llm_calc_app.load()
llm_calc_app.queue()
llm_calc_app.launch()
"""
model_name, input_price, output_price,
multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
source,licence_name,licence_url,languages,release_date,
parameters_estimated,parameters_actual,
open_weight,context,
additional_prices_context_caching,
additional_prices_context_storage,
additional_prices_image_input,additional_prices_image_output,additional_prices_video_input,additional_prices_video_output,additional_prices_audio_input,additional_prices_audio_output,clemscore_v1.6.5_multimodal,clemscore_v1.6.5_ascii,clemscore_v1.6,latency_v1.6,latency_v1.6.5_multimodal,latency_v1.6.5_ascii,
average_clemscore,average_latency,parameters
Final list
model_name, input_price, output_price,
multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
source,licence_name,licence_url,languages,release_date, open_weight,context, average_clemscore,average_latency,parameters
Filter
multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
licence_name+licence_url, languages, release_date, open_weight
RR
model_name, input_price, output_price,
source, release_date
"""