update main data df, minor changes in combined json

app.py

@@ -1,19 +1,74 @@
 import pandas as pd
 import gradio as gr
 import os
+from gradio_rangeslider import RangeSlider
 
 text_leaderboard = pd.read_csv(os.path.join('src', 'main_df.csv'))
+text = "## The range is: {min} to {max}"
+
+
 llm_calc_app = gr.Blocks()
 with llm_calc_app:
 
-    leaderboard_table = gr.Dataframe(
-                            value=text_leaderboard,
-                            elem_id="text-leaderboard-table",
-                            interactive=False,
-                            visible=True,
-                            height=800
-                        )
-    llm_calc_app.load()
+    with gr.Row():
+        """
+        Main Filters Row
+        """
+
+        range_slider = RangeSlider(minimum=0, maximum=100, value=(0, 100))
+        range_ = gr.Markdown(value=text.format(min=0, max=100))
+        range_slider.change(lambda s: text.format(min=s[0], max=s[1]), range_slider, range_,
+                            show_progress="hide", trigger_mode="always_last")
+
+
 
+    with gr.Row():
+        """
+        Main Leaderboard Row
+        """
+
+        leaderboard_table = gr.Dataframe(
+                                value=text_leaderboard,
+                                elem_id="text-leaderboard-table",
+                                interactive=False,
+                                visible=True,
+                                height=800
+                            )
+        
+
+    llm_calc_app.load()
 llm_calc_app.queue()
 llm_calc_app.launch()
+
+
+
+"""
+model_name, input_price, output_price,
+multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
+source,licence_name,licence_url,languages,release_date,
+parameters_estimated,parameters_actual,
+
+open_weight,context, 
+
+additional_prices_context_caching,
+additional_prices_context_storage,
+additional_prices_image_input,additional_prices_image_output,additional_prices_video_input,additional_prices_video_output,additional_prices_audio_input,additional_prices_audio_output,clemscore_v1.6.5_multimodal,clemscore_v1.6.5_ascii,clemscore_v1.6,latency_v1.6,latency_v1.6.5_multimodal,latency_v1.6.5_ascii,
+
+average_clemscore,average_latency,parameters
+
+Final list
+
+model_name, input_price, output_price,
+multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
+source,licence_name,licence_url,languages,release_date, open_weight,context, average_clemscore,average_latency,parameters
+
+
+Filter
+multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,
+licence_name+licence_url, languages, release_date, open_weight
+
+RR
+model_name, input_price, output_price,
+source, release_date
+
+"""

requirements.txt

@@ -1,3 +1,3 @@
 beautifulsoup4==4.12.3
 pandas==2.2.3
-pyppeteer==2.0.0
+gradio_rangeslider==0.0.7

src/combined_data.json

@@ -1,281 +1,305 @@
 [
     {
         "model_name": "Meta-Llama-3-70B-Instruct-hf",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
-        "licence": {
+        "license": {
             "name": "Meta Llama 3 License",
             "url": "https://www.llama.com/llama3/license/" 
         },
-        "languages": ["eng"],
+        "languages": ["en"],
         "release_date": "2024-04-18",
         "parameters":{
             "estimated": false,
-            "actual": "70B"
+            "size": "70B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Meta-Llama-3-8B-Instruct-hf",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct",
-        "licence": {
+        "license": {
             "name": "Meta Llama 3 License",
             "url": "https://www.llama.com/llama3/license/" 
         },
-        "languages": ["eng"],
+        "languages": ["en"],
         "release_date": "2024-04-18",
         "parameters":{
             "estimated": false,
-            "actual": "8B"
+            "size": "8B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Meta-Llama-3.1-405B-Instruct-Turbo",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct"  
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct",
-        "licence": {
+        "license": {
             "name": "Llama 3.1 Community License",
             "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE"
         },
-        "languages": ["eng", "deu", "fra", "ita", "hin", "por", "spa", "tha"],
+        "languages": ["en", "de", "fr", "it", "hi", "pt", "es", "th"],
         "release_date": "2024-07-23",
         "parameters":{
             "estimated": false,
-            "actual": "405B"
+            "size": "405B"
         },
         "open_weight": true,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "Meta-Llama-3.1-70B-Instruct",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct",
-        "licence": {
+        "license": {
             "name": "Llama 3.1 Community License",
             "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE"
         },
-        "languages": ["eng", "deu", "fra", "ita", "hin", "por", "spa", "tha"],
+        "languages": ["en", "de", "fr", "it", "hi", "pt", "es", "th"],
         "release_date": "2024-07-23",
         "parameters": {
             "estimated": false,
-            "actual": "70B"
+            "size": "70B"
         },
         "open_weight": true,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "Meta-Llama-3.1-8B-Instruct",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct",
-        "licence": {
+        "license": {
             "name": "Llama 3.1 Community License",
             "url": "https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE"
         },
-        "languages": ["eng", "deu", "fra", "ita", "hin", "por", "spa", "tha"],
+        "languages": ["en", "de", "fr", "it", "hi", "pt", "es", "th"],
         "release_date": "2024-07-23",
         "parameters": {
             "estimated": false,
-            "actual": "8B"
+            "size": "8B"
         },
         "open_weight": true,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "InternVL2-40B",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/OpenGVLab/InternVL2-40B"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/OpenGVLab/InternVL2-40B",
-        "licence": {
+        "license": {
             "name": "MIT",
             "url": "https://choosealicense.com/licenses/mit/"
         },
-        "languages": ["chi","eng","fre","spa","por","deu","ita","rus","jpn","kor","vie","tha","ara"],
+        "languages": ["zh", "en", "fr", "es", "pt", "de", "it", "ru", "ja", "ko", "vi", "th", "ar"],
         "release_date": "2024-07-15",
         "parameters": {
             "estimated": false,
-            "actual": "40B"
+            "size": "40B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "InternVL2-8B",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/OpenGVLab/InternVL2-8B"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/OpenGVLab/InternVL2-8B",
-        "licence": {
+        "license": {
             "name": "MIT",
             "url": "https://choosealicense.com/licenses/mit/"
         },
-        "languages": ["chi","eng","fre","spa","por","deu","ita","rus","jpn","kor","vie","tha","ara"],
+        "languages": ["zh", "en", "fr", "es", "pt", "de", "it", "ru", "ja", "ko", "vi", "th", "ar"],
         "release_date": "2024-07-15",
         "parameters": {
             "estimated": false,
-            "actual": "8B"
+            "size": "8B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "InternVL2-Llama3-76B",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B",
-        "licence": {
+        "license": {
             "name": "MIT",
             "url": "https://choosealicense.com/licenses/mit/"
         },
-        "languages": ["chi","eng","fre","spa","por","deu","ita","rus","jpn","kor","vie","tha","ara"],
+        "languages": ["zh", "en", "fr", "es", "pt", "de", "it", "ru", "ja", "ko", "vi", "th", "ar"],
         "release_date": "2024-07-15",
         "parameters": {
             "estimated": false,
-            "actual": "76B"
+            "size": "76B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "InternVL2-26B",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/OpenGVLab/InternVL2-26B"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/OpenGVLab/InternVL2-26B",
-        "licence": {
+        "license": {
             "name": "MIT",
             "url": "https://choosealicense.com/licenses/mit/"
         },
-        "languages": ["chi","eng","fre","spa","por","deu","ita","rus","jpn","kor","vie","tha","ara"],
+        "languages": ["zh", "en", "fr", "es", "pt", "de", "it", "ru", "ja", "ko", "vi", "th", "ar"],
         "release_date": "2024-07-15",
         "parameters": {
             "estimated": false,
-            "actual": "26B"
+            "size": "26B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
 
     {
         "model_name": "InternVL2-26B",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/OpenGVLab/InternVL2-26B"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/OpenGVLab/InternVL2-26B",
-        "licence": {
+        "license": {
             "name": "MIT",
             "url": "https://choosealicense.com/licenses/mit/"
         },
-        "languages": ["chi","eng","fre","spa","por","deu","ita","rus","jpn","kor","vie","tha","ara"],
+        "languages": ["zh", "en", "fr", "es", "pt", "de", "it", "ru", "ja", "ko", "vi", "th", "ar"],
         "release_date": "2024-07-15",
         "parameters": {
             "estimated": false,
-            "actual": "26B"
+            "size": "26B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Mistral-Large-Instruct-2407",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng", "fra", "spa", "deu", "ita", "rus", "chi", "jpn", "kor"],
+        "languages": ["en", "fr", "es", "de", "it", "ru", "zh", "ja", "ko"],
         "release_date": "2024-06-12",
         "parameters": {
             "estimated": false,
-            "actual": "70B"
+            "size": "70B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Mixtral-8x22B-Instruct-v0.1",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
@@ -283,283 +307,305 @@
         "release_date": "2024-04-17",
         "parameters": {
             "estimated": false,
-            "actual": "141B"
+            "size": "141B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Mistral-7B-Instruct-v0.2",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng", "fra", "spa", "deu", "ita", "rus", "chi"],
+        "languages": ["en", "fr", "es", "de", "it", "ru", "zh"],
         "release_date": "2024-01-15",
         "parameters": {
             "estimated": false,
-            "actual": "7B"
+            "size": "7B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Mistral-7B-Instruct-v0.1",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng", "fra", "spa", "deu", "ita", "rus", "chi"],
+        "languages": ["en", "fr", "es", "de", "it", "ru", "zh"],
         "release_date": "2023-12-11",
         "parameters": {
             "estimated": false,
-            "actual": "7B"
+            "size": "7B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "Mixtral-8x7B-Instruct-v0.1",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng", "fra", "spa", "deu", "ita", "rus"],
+        "languages": ["en", "fr", "es", "de", "it", "ru"],
         "release_date": "2023-12-11",
         "parameters": {
             "estimated": false,
-            "actual": "46.7B"
+            "size": "46.7B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "openchat-3.5-0106",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/openchat/openchat-3.5-0106"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/openchat/openchat-3.5-0106",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng"],
+        "languages": ["en"],
         "release_date": "2024-01-06",
         "parameters": {
             "estimated": false,
-            "actual": "7B"
+            "size": "7B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "openchat-3.5-1210",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/openchat/openchat-3.5-1210"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/openchat/openchat-3.5-1210",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng"],
+        "languages": ["en"],
         "release_date": "2023-12-10",
         "parameters": {
             "estimated": false,
-            "actual": "7B"
+            "size": "7B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "openchat_3.5",
-        "input_price": "0",
-        "output_price": "0",
+        "pricing": {
+            "input_price": "0$",
+            "output_price": "0$",
+            "source": "https://huggingface.co/openchat/openchat_3.5"
+        },
         "multimodality": {
             "image": false,
             "multiple_image": false,
             "audio": false,
             "video": false
         },
-        "source": "https://huggingface.co/openchat/openchat_3.5",
-        "licence": {
+        "license": {
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0"
         },
-        "languages": ["eng"],
+        "languages": ["en"],
         "release_date": "2023-10-30",
         "parameters": {
             "estimated": false,
-            "actual": "7B"
+            "size": "7B"
         },
         "open_weight": true,
-        "context": "8k"
+        "context_size": "8k"
     },
     {
         "model_name": "gpt-4o-mini-2024-07-18",
-        "input_price": "$0.150",
-        "output_price": "$0.600",
+        "pricing": {
+            "input_price": "$0.150",
+            "output_price": "$0.600",
+            "source": "https://openai.com/api/pricing/"
+            },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://openai.com/api/pricing/",
-        "licence": {
+        "license": {
             "name": "Commercial License",
             "url": "https://openai.com/policies/terms-of-use"
         },
-        "languages": ["eng", "spa", "fra", "deu", "zho", "zht", "jpn", "kor", "ita", "por", "nld", "rus", "ara", "hin", "tur", "vie", "pol", "tha", "swe", "dan", "nor", "fin", "hun", "ces", "slk", "rom", "bul", "ukr", "lit", "lav", "est", "slv", "msa", "ind", "fil", "swz", "amh"],
+        "languages": ["en", "es", "fr", "de", "zh", "ja", "ko", "it", "pt", "nl", "ru", "ar", "hi", "tr", "vi", "pl", "th", "sv", "da", "no", "fi", "hu", "cs", "sk", "ro", "bg", "uk", "lt", "lv", "et", "sl", "ms", "id", "tl", "sw", "am"],
         "release_date": "2024-07-18",
         "parameters": {
             "estimated": true,
-            "actual": "8B"
+            "size": "8B"
         },
         "open_weight": false,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "gpt-4o-2024-08-06",
-        "input_price": "$2.50",
-        "output_price": "$10.00",
+        "pricing": {
+            "input_price": "$2.50",
+            "output_price": "$10.00",
+            "source": "https://openai.com/api/pricing/"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://openai.com/api/pricing/",
-        "licence": {
+        "license": {
             "name": "Commercial License",
             "url": "https://openai.com/policies/terms-of-use"
         },
-        "languages": ["eng", "spa", "fra", "deu", "zho", "zht", "jpn", "kor", "ita", "por", "nld", "rus", "ara", "hin", "tur", "vie", "pol", "tha", "swe", "dan", "nor", "fin", "hun", "ces", "slk", "rom", "bul", "ukr", "lit", "lav", "est", "slv", "msa", "ind", "fil", "swz", "amh"],
+        "languages": ["en", "es", "fr", "de", "zh", "ja", "ko", "it", "pt", "nl", "ru", "ar", "hi", "tr", "vi", "pl", "th", "sv", "da", "no", "fi", "hu", "cs", "sk", "ro", "bg", "uk", "lt", "lv", "et", "sl", "ms", "id", "tl", "sw", "am"],
         "release_date": "2024-08-06",
         "parameters": {
             "estimated": true,
-            "actual": "200B"
+            "size": "200B"
         },
         "open_weight": false,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "gpt-4o-2024-05-13",
-        "input_price": "$2.50",
-        "output_price": "$10.00",
+        "pricing":{
+            "input_price": "$2.50",
+            "output_price": "$10.00",
+            "source": "https://openai.com/api/pricing/"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://openai.com/api/pricing/",
-        "licence": {
+        "license": {
             "name": "Commercial License",
             "url": "https://openai.com/policies/terms-of-use"
         },
-        "languages": ["eng", "spa", "fra", "deu", "zho", "zht", "jpn", "kor", "ita", "por", "nld", "rus", "ara", "hin", "tur", "vie", "pol", "tha", "swe", "dan", "nor", "fin", "hun", "ces", "slk", "rom", "bul", "ukr", "lit", "lav", "est", "slv", "msa", "ind", "fil", "swz", "amh"],
+        "languages": ["en", "es", "fr", "de", "zh", "ja", "ko", "it", "pt", "nl", "ru", "ar", "hi", "tr", "vi", "pl", "th", "sv", "da", "no", "fi", "hu", "cs", "sk", "ro", "bg", "uk", "lt", "lv", "et", "sl", "ms", "id", "tl", "sw", "am"],
         "release_date": "2024-05-13",
         "parameters": {
             "estimated": true,
-            "actual": "200B"
+            "size": "200B"
         },
         "open_weight": false,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "gpt-4-1106-vision-preview",
-        "input_price": "$10.00",
-        "output_price": "$30.00",
+        "pricing": {
+            "input_price": "$10.00",
+            "output_price": "$30.00",
+            "source": "https://openai.com/api/pricing/"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": false,
             "video": false
         },
-        "source": "https://openai.com/api/pricing/",
-        "licence": {
+        "license": {
             "name": "Commercial License",
             "url": "https://openai.com/policies/terms-of-use"
         },
-        "languages": ["eng", "spa", "fra", "deu", "zho", "zht", "jpn", "kor", "ita", "por", "nld", "rus", "ara", "hin", "tur", "vie", "pol", "tha", "swe", "dan", "nor", "fin", "hun", "ces", "slk", "rom", "bul", "ukr", "lit", "lav", "est", "slv", "msa", "ind", "fil", "swz", "amh"],
+        "languages": ["en", "es", "fr", "de", "zh", "ja", "ko", "it", "pt", "nl", "ru", "ar", "hi", "tr", "vi", "pl", "th", "sv", "da", "no", "fi", "hu", "cs", "sk", "ro", "bg", "uk", "lt", "lv", "et", "sl", "ms", "id", "tl", "sw", "am"],
         "release_date": "2023-11-06",
         "parameters": {
-            "estimated": "1.76T",
-            "actual": false
+            "estimated": true,
+            "size": "1760B"
         },
         "open_weight": false,
-        "context": "128k"
+        "context_size": "128k"
     },
     {
         "model_name": "gemini-1.5-flash-latest",
-        "input_price": "$0.075",
-        "output_price": "$0.30",
-        "additional_prices": {
-            "context_caching": "$0.01875",
-            "context_storage": "$1.00",
-            "image_input": "$0.02",
-            "image_output": "$0.04",
-            "video_input": "$0.02",
-            "video_output": "$0.04",
-            "audio_input": "$0.02",
-            "audio_output": "$0.04"
-        }, 
+        "pricing":{
+            "input_price": "$0.075",
+            "output_price": "$0.30",
+            "additional_prices": {
+                "context_caching": "$0.01875",
+                "context_storage": "$1.00",
+                "image_input": "$0.02",
+                "image_output": "$0.04",
+                "video_input": "$0.02",
+                "video_output": "$0.04",
+                "audio_input": "$0.02",
+                "audio_output": "$0.04"
+            },
+            "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
+        },
         "multimodality": {
             "image": true,
             "multiple_image": true,
             "audio": true,
             "video": true
-        },
-        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
-        "licence": {
+        },       
+        "license": {
             "name": "Commercial License",
             "url": ""
         },
-        "languages": ["lit", "nor", "pol", "por", "ron", "rus", "srp", "slk", "slv", "spa", "swa", "swe", "tha", "tur", "ukr", "vie", "zho", "hrv", "ces", "dan", "nld", "eng", "est", "fin", "fra", "deu", "ell", "heb", "hin", "hun", "ind", "ita", "jpn", "kor", "lav", "ara", "ben", "bul"],
+        "languages": ["lt", "no", "pl", "pt", "ro", "ru", "sr", "sk", "sl", "es", "sw", "sv", "th", "tr", "uk", "vi", "zh", "hr", "cs", "da", "nl", "en", "et", "fi", "fr", "de", "el", "he", "hi", "hu", "id", "it", "ja", "ko", "lv", "ar", "bn", "bg"],
         "release_date": "2024-05-24",
         "parameters": {
-            "estimated": false,
-            "actual": false
+            "estimated": true,
+            "size": "1760B"
         },
         "open_weight": false,
-        "context": ">128k"
+        "context_size": "128k"
     }
 ]

src/main_df.csv

@@ -1,24 +1,24 @@
-model_name,input_price,output_price,multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,source,licence_name,licence_url,languages,release_date,parameters_estimated,parameters_actual,open_weight,context,additional_prices_context_caching,additional_prices_context_storage,additional_prices_image_input,additional_prices_image_output,additional_prices_video_input,additional_prices_video_output,additional_prices_audio_input,additional_prices_audio_output,clemscore_v1.6.5_multimodal,clemscore_v1.6.5_ascii,clemscore_v1.6
-Meta-Llama-3-70B-Instruct-hf,0,0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct,Meta Llama 3 License,https://www.llama.com/llama3/license/,eng,2024-04-18,False,70B,True,8k,,,,,,,,,0.0,0.0,35.11
-Meta-Llama-3-8B-Instruct-hf,0,0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct,Meta Llama 3 License,https://www.llama.com/llama3/license/,eng,2024-04-18,False,8B,True,8k,,,,,,,,,0.0,0.0,19.99
-Meta-Llama-3.1-405B-Instruct-Turbo,0,0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"eng, deu, fra, ita, hin, por, spa, tha",2024-07-23,False,405B,True,128k,,,,,,,,,0.0,0.0,52.11
-Meta-Llama-3.1-70B-Instruct,0,0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"eng, deu, fra, ita, hin, por, spa, tha",2024-07-23,False,70B,True,128k,,,,,,,,,0.0,0.0,38.83
-Meta-Llama-3.1-8B-Instruct,0,0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"eng, deu, fra, ita, hin, por, spa, tha",2024-07-23,False,8B,True,128k,,,,,,,,,0.0,0.0,18.36
-InternVL2-40B,0,0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-40B,MIT,https://choosealicense.com/licenses/mit/,"chi, eng, fre, spa, por, deu, ita, rus, jpn, kor, vie, tha, ara",2024-07-15,False,40B,True,8k,,,,,,,,,32.23,33.2,0.0
-InternVL2-8B,0,0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-8B,MIT,https://choosealicense.com/licenses/mit/,"chi, eng, fre, spa, por, deu, ita, rus, jpn, kor, vie, tha, ara",2024-07-15,False,8B,True,8k,,,,,,,,,23.17,36.05,0.0
-InternVL2-Llama3-76B,0,0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B,MIT,https://choosealicense.com/licenses/mit/,"chi, eng, fre, spa, por, deu, ita, rus, jpn, kor, vie, tha, ara",2024-07-15,False,76B,True,8k,,,,,,,,,33.84,43.29,0.0
-InternVL2-26B,0,0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,https://choosealicense.com/licenses/mit/,"chi, eng, fre, spa, por, deu, ita, rus, jpn, kor, vie, tha, ara",2024-07-15,False,26B,True,8k,,,,,,,,,37.45,32.27,0.0
-InternVL2-26B,0,0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,https://choosealicense.com/licenses/mit/,"chi, eng, fre, spa, por, deu, ita, rus, jpn, kor, vie, tha, ara",2024-07-15,False,26B,True,8k,,,,,,,,,37.45,32.27,0.0
-Mistral-Large-Instruct-2407,0,0,False,False,False,False,https://huggingface.co/mistralai/Mistral-Large-Instruct-2407,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"eng, fra, spa, deu, ita, rus, chi, jpn, kor",2024-06-12,False,70B,True,8k,,,,,,,,,0.0,0.0,45.39
-Mixtral-8x22B-Instruct-v0.1,0,0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"eng, fra, spa, deu, ita, rus",2024-04-17,False,141B,True,8k,,,,,,,,,0.0,0.0,12.69
-Mistral-7B-Instruct-v0.2,0,0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"eng, fra, spa, deu, ita, rus, chi",2024-01-15,False,7B,True,8k,,,,,,,,,0.0,0.0,9.75
-Mistral-7B-Instruct-v0.1,0,0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"eng, fra, spa, deu, ita, rus, chi",2023-12-11,False,7B,True,8k,,,,,,,,,0.0,0.0,8.01
-Mixtral-8x7B-Instruct-v0.1,0,0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"eng, fra, spa, deu, ita, rus",2023-12-11,False,46.7B,True,8k,,,,,,,,,0.0,0.0,8.17
-openchat-3.5-0106,0,0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-0106,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,eng,2024-01-06,False,7B,True,8k,,,,,,,,,0.0,0.0,17.1
-openchat-3.5-1210,0,0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-1210,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,eng,2023-12-10,False,7B,True,8k,,,,,,,,,0.0,0.0,18.22
-openchat_3.5,0,0,False,False,False,False,https://huggingface.co/openchat/openchat_3.5,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,eng,2023-10-30,False,7B,True,8k,,,,,,,,,0.0,0.0,23.64
-gpt-4o-mini-2024-07-18,$0.150,$0.600,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"eng, spa, fra, deu, zho, zht, jpn, kor, ita, por, nld, rus, ara, hin, tur, vie, pol, tha, swe, dan, nor, fin, hun, ces, slk, rom, bul, ukr, lit, lav, est, slv, msa, ind, fil, swz, amh",2024-07-18,True,8B,False,128k,,,,,,,,,58.46,63.87,34.64
-gpt-4o-2024-08-06,$2.50,$10.00,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"eng, spa, fra, deu, zho, zht, jpn, kor, ita, por, nld, rus, ara, hin, tur, vie, pol, tha, swe, dan, nor, fin, hun, ces, slk, rom, bul, ukr, lit, lav, est, slv, msa, ind, fil, swz, amh",2024-08-06,True,200B,False,128k,,,,,,,,,80.04,80.96,47.71
-gpt-4o-2024-05-13,$2.50,$10.00,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"eng, spa, fra, deu, zho, zht, jpn, kor, ita, por, nld, rus, ara, hin, tur, vie, pol, tha, swe, dan, nor, fin, hun, ces, slk, rom, bul, ukr, lit, lav, est, slv, msa, ind, fil, swz, amh",2024-05-13,True,200B,False,128k,,,,,,,,,69.56,82.72,48.34
-gpt-4-1106-vision-preview,$10.00,$30.00,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"eng, spa, fra, deu, zho, zht, jpn, kor, ita, por, nld, rus, ara, hin, tur, vie, pol, tha, swe, dan, nor, fin, hun, ces, slk, rom, bul, ukr, lit, lav, est, slv, msa, ind, fil, swz, amh",2023-11-06,1.76T,False,False,128k,,,,,,,,,73.55,68.14,0.0
-gemini-1.5-flash-latest,$0.075,$0.30,True,True,True,True,https://cloud.google.com/vertex-ai/generative-ai/pricing,Commercial License,,"lit, nor, pol, por, ron, rus, srp, slk, slv, spa, swa, swe, tha, tur, ukr, vie, zho, hrv, ces, dan, nld, eng, est, fin, fra, deu, ell, heb, hin, hun, ind, ita, jpn, kor, lav, ara, ben, bul",2024-05-24,False,False,False,>128k,$0.01875,$1.00,$0.02,$0.04,$0.02,$0.04,$0.02,$0.04,47.73,47.88,32.0
+model_name,input_price,output_price,multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,source,license_name,license_url,languages,release_date,open_weight,context_size,average_clemscore,average_latency,parameter_size,estimated
+Meta-Llama-3-70B-Instruct-hf,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct,Meta Llama 3 License,https://www.llama.com/llama3/license/,en,2024-04-18,True,8,11.703333333333333,1.1160853862207483,70.0,False
+Meta-Llama-3-8B-Instruct-hf,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct,Meta Llama 3 License,https://www.llama.com/llama3/license/,en,2024-04-18,True,8,6.663333333333333,0.7054825144189354,8.0,False
+Meta-Llama-3.1-405B-Instruct-Turbo,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"en, de, fr, it, hi, pt, es, th",2024-07-23,True,128,17.37,0.2628701315515277,405.0,False
+Meta-Llama-3.1-70B-Instruct,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"en, de, fr, it, hi, pt, es, th",2024-07-23,True,128,12.943333333333333,0.27016850919817575,70.0,False
+Meta-Llama-3.1-8B-Instruct,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"en, de, fr, it, hi, pt, es, th",2024-07-23,True,128,6.12,0.06876858280202812,8.0,False
+InternVL2-40B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-40B,MIT,https://choosealicense.com/licenses/mit/,"zh, en, fr, es, pt, de, it, ru, ja, ko, vi, th, ar",2024-07-15,True,8,21.810000000000002,2.609271782765464,40.0,False
+InternVL2-8B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-8B,MIT,https://choosealicense.com/licenses/mit/,"zh, en, fr, es, pt, de, it, ru, ja, ko, vi, th, ar",2024-07-15,True,8,19.74,0.8367998047485775,8.0,False
+InternVL2-Llama3-76B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B,MIT,https://choosealicense.com/licenses/mit/,"zh, en, fr, es, pt, de, it, ru, ja, ko, vi, th, ar",2024-07-15,True,8,25.709999999999997,4.591395944741546,76.0,False
+InternVL2-26B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,https://choosealicense.com/licenses/mit/,"zh, en, fr, es, pt, de, it, ru, ja, ko, vi, th, ar",2024-07-15,True,8,23.24,1.7593004986949285,26.0,False
+InternVL2-26B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,https://choosealicense.com/licenses/mit/,"zh, en, fr, es, pt, de, it, ru, ja, ko, vi, th, ar",2024-07-15,True,8,23.24,1.7593004986949285,26.0,False
+Mistral-Large-Instruct-2407,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-Large-Instruct-2407,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"en, fr, es, de, it, ru, zh, ja, ko",2024-06-12,True,8,15.13,0.41482225628780656,70.0,False
+Mixtral-8x22B-Instruct-v0.1,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"eng, fra, spa, deu, ita, rus",2024-04-17,True,8,4.2299999999999995,0.3586451521191292,141.0,False
+Mistral-7B-Instruct-v0.2,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"en, fr, es, de, it, ru, zh",2024-01-15,True,8,3.25,0.25450503989030154,7.0,False
+Mistral-7B-Instruct-v0.1,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"en, fr, es, de, it, ru, zh",2023-12-11,True,8,2.67,0.09428825169239076,7.0,False
+Mixtral-8x7B-Instruct-v0.1,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"en, fr, es, de, it, ru",2023-12-11,True,8,2.723333333333333,0.31309892202121054,46.7,False
+openchat-3.5-0106,0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-0106,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,en,2024-01-06,True,8,5.7,0.09736504835188847,7.0,False
+openchat-3.5-1210,0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-1210,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,en,2023-12-10,True,8,6.073333333333333,0.09349942563676568,7.0,False
+openchat_3.5,0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat_3.5,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,en,2023-10-30,True,8,7.88,0.10576256228206875,7.0,False
+gpt-4o-mini-2024-07-18,0.15,0.6,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"en, es, fr, de, zh, ja, ko, it, pt, nl, ru, ar, hi, tr, vi, pl, th, sv, da, no, fi, hu, cs, sk, ro, bg, uk, lt, lv, et, sl, ms, id, tl, sw, am",2024-07-18,False,128,52.32333333333333,1.619222935116773,8.0,True
+gpt-4o-2024-08-06,2.5,10.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"en, es, fr, de, zh, ja, ko, it, pt, nl, ru, ar, hi, tr, vi, pl, th, sv, da, no, fi, hu, cs, sk, ro, bg, uk, lt, lv, et, sl, ms, id, tl, sw, am",2024-08-06,False,128,69.57000000000001,1.5771123003908176,200.0,True
+gpt-4o-2024-05-13,2.5,10.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"en, es, fr, de, zh, ja, ko, it, pt, nl, ru, ar, hi, tr, vi, pl, th, sv, da, no, fi, hu, cs, sk, ro, bg, uk, lt, lv, et, sl, ms, id, tl, sw, am",2024-05-13,False,128,66.87333333333333,3.704921340164487,200.0,True
+gpt-4-1106-vision-preview,10.0,30.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"en, es, fr, de, zh, ja, ko, it, pt, nl, ru, ar, hi, tr, vi, pl, th, sv, da, no, fi, hu, cs, sk, ro, bg, uk, lt, lv, et, sl, ms, id, tl, sw, am",2023-11-06,False,128,47.23,2.217200177676117,1760.0,True
+gemini-1.5-flash-latest,0.075,0.3,True,True,True,True,https://cloud.google.com/vertex-ai/generative-ai/pricing,Commercial License,,"lt, no, pl, pt, ro, ru, sr, sk, sl, es, sw, sv, th, tr, uk, vi, zh, hr, cs, da, nl, en, et, fi, fr, de, el, he, hi, hu, id, it, ja, ko, lv, ar, bn, bg",2024-05-24,False,128,42.53666666666667,26.268280234692302,1760.0,True

src/process_data.py

@@ -11,30 +11,31 @@ flattened_data = []
 for entry in data:
     flattened_entry = {
         "model_name": entry["model_name"],
-        "input_price": entry["input_price"],
-        "output_price": entry["output_price"],
+        "input_price": entry["pricing"]["input_price"],
+        "output_price": entry["pricing"]["output_price"],
         "multimodality_image": entry["multimodality"]["image"],
         "multimodality_multiple_image": entry["multimodality"]["multiple_image"],
         "multimodality_audio": entry["multimodality"]["audio"],
         "multimodality_video": entry["multimodality"]["video"],
-        "source": entry["source"],
-        "licence_name": entry["licence"]["name"],
-        "licence_url": entry["licence"]["url"],
+        "source": entry["pricing"]["source"],
+        "license_name": entry["license"]["name"],
+        "license_url": entry["license"]["url"],
         "languages": ", ".join(entry["languages"]),
         "release_date": entry["release_date"],
-        "parameters_estimated": entry["parameters"]["estimated"],
-        "parameters_actual": entry["parameters"]["actual"],
+        "parameter_size": entry["parameters"]["size"],
+        "estimated": entry["parameters"]["estimated"],
         "open_weight": entry["open_weight"],
-        "context": entry["context"],
+        "context_size": entry["context_size"],
+
         # ... additional prices ...
-        "additional_prices_context_caching": entry.get("additional_prices", {}).get("context_caching", None),
-        "additional_prices_context_storage": entry.get("additional_prices", {}).get("context_storage", None),
-        "additional_prices_image_input": entry.get("additional_prices", {}).get("image_input", None),
-        "additional_prices_image_output": entry.get("additional_prices", {}).get("image_output", None),
-        "additional_prices_video_input": entry.get("additional_prices", {}).get("video_input", None),
-        "additional_prices_video_output": entry.get("additional_prices", {}).get("video_output", None),
-        "additional_prices_audio_input": entry.get("additional_prices", {}).get("audio_input", None),
-        "additional_prices_audio_output": entry.get("additional_prices", {}).get("audio_output", None),
+        "additional_prices_context_caching": entry["pricing"].get("additional_prices", {}).get("context_caching", None),
+        "additional_prices_context_storage": entry["pricing"].get("additional_prices", {}).get("context_storage", None),
+        "additional_prices_image_input": entry["pricing"].get("additional_prices", {}).get("image_input", None),
+        "additional_prices_image_output": entry["pricing"].get("additional_prices", {}).get("image_output", None),
+        "additional_prices_video_input": entry["pricing"].get("additional_prices", {}).get("video_input", None),
+        "additional_prices_video_output": entry["pricing"].get("additional_prices", {}).get("video_output", None),
+        "additional_prices_audio_input": entry["pricing"].get("additional_prices", {}).get("audio_input", None),
+        "additional_prices_audio_output": entry["pricing"].get("additional_prices", {}).get("audio_output", None),
     }
     flattened_data.append(flattened_entry)
 
@@ -57,12 +58,80 @@ clemscore_map_1_6_5_multimodal = dict(zip(results_1_6_5_multimodal[0], results_1
 clemscore_map_1_6_5_ascii = dict(zip(results_1_6_5_ascii[0], results_1_6_5_ascii[1]))
 clemscore_map_1_6 = dict(zip(results_1_6[0], results_1_6[1]))
 
-print(clemscore_map_1_6)
 
 # Add clemscore columns to the main DataFrame
 df['clemscore_v1.6.5_multimodal'] = df['model_name'].map(clemscore_map_1_6_5_multimodal).fillna(0).astype(float)
 df['clemscore_v1.6.5_ascii'] = df['model_name'].map(clemscore_map_1_6_5_ascii).fillna(0).astype(float)
 df['clemscore_v1.6'] = df['model_name'].map(clemscore_map_1_6).fillna(0).astype(float)
 
+# Load the latency CSV files
+latency_1_6 = pd.read_csv('src/v1.6_latency.csv', header=None)
+latency_1_6_5_ascii = pd.read_csv('src/v1.6.5_ascii_latency.csv', header=None)
+latency_1_6_5_multimodal = pd.read_csv('src/v1.6.5_multimodal_latency.csv', header=None)
+
+# Create a mapping for latency values
+latency_map_1_6 = dict(zip(latency_1_6[0], latency_1_6[1]))
+latency_map_1_6_5_ascii = dict(zip(latency_1_6_5_ascii[0], latency_1_6_5_ascii[1]))
+latency_map_1_6_5_multimodal = dict(zip(latency_1_6_5_multimodal[0], latency_1_6_5_multimodal[1]))
+
+# Add latency columns to the main DataFrame
+df['latency_v1.6'] = df['model_name'].map(latency_map_1_6).fillna(0).astype(float)
+df['latency_v1.6.5_multimodal'] = df['model_name'].map(latency_map_1_6_5_multimodal).fillna(0).astype(float)
+df['latency_v1.6.5_ascii'] = df['model_name'].map(latency_map_1_6_5_ascii).fillna(0).astype(float)
+
+
+# Calculate average latency and clemscore
+df['average_clemscore'] = df[['clemscore_v1.6.5_multimodal', 'clemscore_v1.6.5_ascii', 'clemscore_v1.6']].mean(axis=1)
+df['average_latency'] = df[['latency_v1.6', 'latency_v1.6.5_multimodal', 'latency_v1.6.5_ascii']].mean(axis=1)
+
+
+# More clean up
+# Clean and convert prices to float
+df['input_price'] = df['input_price'].replace({'\$': '', '': None}, regex=True).astype(float)
+df['output_price'] = df['output_price'].replace({'\$': '', '': None}, regex=True).astype(float)
+
+# Clean and convert additional prices to float
+additional_price_columns = [
+    'additional_prices_context_caching',
+    'additional_prices_context_storage',
+    'additional_prices_image_input',
+    'additional_prices_image_output',
+    'additional_prices_video_input',
+    'additional_prices_video_output',
+    'additional_prices_audio_input',
+    'additional_prices_audio_output'
+]
+
+for col in additional_price_columns:
+    df[col] = df[col].replace({'\$': '', '': None}, regex=True).astype(float)
+
+# Clean and convert context to integer
+df['context_size'] = df['context_size'].replace({'k': ''}, regex=True).astype(int)
+
+df['parameter_size'] = df['parameter_size'].replace({'B': '', '': None}, regex=True).astype(float)
+
+# Keep only the specified columns
+df = df[[
+    'model_name', 
+    'input_price', 
+    'output_price',
+    'multimodality_image',
+    'multimodality_multiple_image',
+    'multimodality_audio',
+    'multimodality_video',
+    'source',
+    'license_name',
+    'license_url',
+    'languages',
+    'release_date', 
+    'open_weight',
+    'context_size', 
+    'average_clemscore',
+    'average_latency',
+    'parameter_size',
+    'estimated'
+]]
+
+
 # Save to CSV
 df.to_csv('src/main_df.csv', index=False)

src/v1.6.5_ascii_latency.csv

@@ -0,0 +1,19 @@
+model,latency
+InternVL2-Llama3-76B,3.114070534839201
+InternVL2-26B,1.0386292812723366
+gpt-4o-2024-05-13,1.0694715724240909
+claude-3-5-sonnet-20240620,2.1419643781420827
+idefics-9b-instruct,1.4799789114093953
+claude-3-opus-20240229,2.6997723769014104
+gpt-4o-mini-2024-07-18,0.6847286470238105
+gpt-4-1106-vision-preview,1.9390430972762676
+Phi-3.5-vision-instruct,0.9667233222621183
+InternVL2-40B,1.5607129299048943
+gpt-4o-2024-08-06,0.8286696862643078
+idefics-80b-instruct,9.909203573061212
+InternVL2-8B,0.5617990863945588
+gemini-1.5-flash-latest,13.396903850443566
+Idefics3-8B-Llama3,0.5553913162090334
+Phi-3-vision-128k-instruct,1.1679793685649222
+Pixtral-12B-2409,0.3775700848214281
+internlm-xcomposer2d5-7b,6.095479326235735

src/v1.6.5_multimodal_latency.csv

@@ -0,0 +1,20 @@
+model,latency
+InternVL2-Llama3-76B,10.660117299385437
+InternVL2-26B,4.239272214812449
+dolphin-vision-72b,10.19095800373974
+gpt-4o-2024-05-13,9.488193374830397
+claude-3-5-sonnet-20240620,3.2490840805235996
+idefics-9b-instruct,4.156911970172689
+claude-3-opus-20240229,4.8763568649807185
+gpt-4o-mini-2024-07-18,3.638671743317612
+gpt-4-1106-vision-preview,4.712557435752083
+Phi-3.5-vision-instruct,1.5404880504707106
+InternVL2-40B,6.267102418391499
+gpt-4o-2024-08-06,3.3857084617187416
+idefics-80b-instruct,6.808930391550246
+InternVL2-8B,1.9486003278511734
+gemini-1.5-flash-latest,28.203669643584554
+Idefics3-8B-Llama3,2.7247848158020056
+Phi-3-vision-128k-instruct,1.3368420310828857
+Pixtral-12B-2409,1.4976731684122302
+internlm-xcomposer2d5-7b,8.438096179522184

src/v1.6_latency.csv

@@ -0,0 +1,69 @@
+model,latency
+Phi-3-mini-128k-instruct,0.6615315832127351
+Meta-Llama-3.1-70B-Instruct,0.8105055275945272
+Meta-Llama-3.1-405B-Instruct-Turbo,0.7886103946545832
+mistral-medium-2312,3.3167870515212137
+gemma-2-2b-it,0.3139821517919896
+Meta-Llama-3-8B-Instruct-hf,2.116447543256806
+Qwen1.5-1.8B-Chat,0.5502705450000004
+CodeLlama-34b-Instruct-hf,3.851887315425931
+Qwen2-72B-Instruct,0.9480584860151365
+Meta-Llama-3.1-8B-Instruct,0.20630574840608434
+gemma-1.1-7b-it,0.1782953878345513
+gemma-2-27b-it,0.9922771009345804
+openchat-3.5-1210,0.28049827691029705
+codegemma-7b-it,0.30489740508652785
+dolphin-2.5-mixtral-8x7b,1.100729847312237
+sheep-duck-llama-2-70b-v1.1,5.524607914346896
+gpt-4-0613,0.6484411465828742
+Meta-Llama-3-70B-Instruct-hf,3.348256158662245
+openchat_3.5,0.31728768684620623
+Yi-34B-Chat,1.2871676207135385
+gpt-4o-2024-05-13,0.5570990732389735
+openchat-3.5-0106,0.2920951450556654
+Mixtral-8x7B-Instruct-v0.1,0.9392967660636317
+gpt-4,0.9930876319528372
+Qwen1.5-14B-Chat,0.37289333481152975
+mistral-large-2402,0.3967416598893948
+claude-3-5-sonnet-20240620,0.879929281888541
+gemma-7b-it,0.6112263564356414
+gpt-4-0125-preview,1.0418927523113601
+gpt-4-1106-preview,0.7767265743542753
+gemini-1.5-pro-latest,82.90283880578006
+Nous-Hermes-2-Mixtral-8x7B-SFT,0.860892263303385
+claude-2.1,1.6836316221022491
+gemma-1.1-2b-it,0.11925699461279458
+claude-3-opus-20240229,2.955845827917778
+command-r,0.18832414914586068
+gpt-4o-mini-2024-07-18,0.5342684150088973
+sheep-duck-llama-2-13b,2.9462099794520453
+SUS-Chat-34B,2.27951476106911
+claude-3-haiku-20240307,0.869549739619107
+gpt-4o-2024-08-06,0.5169587531894035
+Yi-1.5-34B-Chat,9.232551962889337
+Qwen2-7B-Instruct,0.3589407217948713
+Mixtral-8x22B-Instruct-v0.1,1.0759354563573875
+vicuna-13b-v1.5,1.4753938719676492
+gemma-2-9b-it,0.3692553324432539
+Mistral-7B-Instruct-v0.2,0.7635151196709047
+Qwen1.5-32B-Chat,0.5092292557397938
+Qwen1.5-0.5B-Chat,0.173469139375476
+Starling-LM-7B-beta,1.365002297029707
+gemini-1.0-pro,0.6346876567117742
+gemini-1.5-flash-latest,37.2042672100488
+aya-23-35B,0.5755088395104249
+llama-2-70b-chat-hf,4.724659620079609
+Yi-1.5-9B-Chat,8.02352422018858
+WizardLM-70b-v1.0,3.9249772038834863
+Yi-1.5-6B-Chat,6.496741103848931
+tulu-2-dpo-70b,7.848597339328551
+Qwen1.5-72B-Chat,12.689668927658234
+Mistral-7B-Instruct-v0.1,0.2828647550771723
+gpt-3.5,0.6829601016193744
+aya-23-8B,0.4818848185613409
+Qwen1.5-7B-Chat,0.38989076908838965
+claude-3-sonnet-20240229,1.419486012822594
+vicuna-33b-v1.3,0.8235025152162343
+WizardLM-13b-v1.2,3.565436762576296
+Mistral-Large-Instruct-2407,1.2444667688634197
+command-r-plus,0.3104016019283745