This leaderboard introduces generative tasks evaluation for Arabic Large Language Models (LLMs). Powered by the new 3C3H evaluation measure, this framework delivers a transparent, robust, and holistic evaluation system that balances factual accuracy and usability assessment for a production ready setting.
For more details, please consider going through the technical blogpost here.
""" ABOUT_SECTION = """ ## About The AraGen Leaderboard is designed to evaluate and compare the performance of Chat Arabic Large Language Models (LLMs) on a set of generative tasks. By leveraging the new **3C3H** evaluation measure which evaluate the model's output across six dimensions —Correctness, Completeness, Conciseness, Helpfulness, Honesty, and Harmlessness— the leaderboard provides a comprehensive and holistic evaluation of a model's performance in generating human-like and ethically responsible content. ### Why Focus on Chat Models? AraGen Leaderboard —And 3C3H in general— is specifically designed to assess **chat models**, which interact in conversational settings, intended for end user interaction and require a blend of factual accuracy and user-centric dialogue capabilities. While it is technically possible to submit foundational models, we kindly ask users to refrain from doing so. For evaluations of foundational models using likelihood accuracy based benchmarks, please refer to the [Open Arabic LLM Leaderboard (OALL)](https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard). ### How to Submit Your Model? Navigate to the submission section below to submit your open chat model from the HuggingFace Hub for evaluation. Ensure that your model is public and the submmited metadata (precision, revision, #params) is accurate. ### Contact For any inquiries or assistance, feel free to reach out through the community tab at [Inception AraGen Community](https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard/discussions) or via [email](mailto:ali.filali@inceptionai.ai). """ CITATION_BUTTON_LABEL = """ Copy the following snippet to cite these results """ CITATION_BUTTON_TEXT = """ @misc{AraGen, author = {El Filali, Ali and Sengupta, Neha and Abouelseoud, Arwa and Nakov, Preslav and Fourrier, Clémentine}, title = {Rethinking LLM Evaluation with 3C3H: AraGen Benchmark and Leaderboard}, year = {2024}, publisher = {Inception}, howpublished = "url{https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard}" } """ def load_results(): # Get the current directory of the script and construct the path to results.json current_dir = os.path.dirname(os.path.abspath(__file__)) results_file = os.path.join(current_dir, "assets", "results", "results.json") # Load the JSON data from the specified file with open(results_file, 'r') as f: data = json.load(f) # Filter out any entries that only contain '_last_sync_timestamp' filtered_data = [] for entry in data: # If '_last_sync_timestamp' is the only key, skip it if len(entry.keys()) == 1 and "_last_sync_timestamp" in entry: continue filtered_data.append(entry) data = filtered_data # Lists to collect data data_3c3h = [] data_tasks = [] for model_data in data: # Extract model meta data meta = model_data.get('Meta', {}) model_name = meta.get('Model Name', 'UNK') revision = meta.get('Revision', 'UNK') precision = meta.get('Precision', 'UNK') params = meta.get('Params', 'UNK') license = meta.get('License', 'UNK') # Convert "Model Size" to numeric, treating "UNK" as infinity try: model_size_numeric = float(params) except (ValueError, TypeError): model_size_numeric = np.inf # 3C3H Scores scores_data = model_data.get('claude-3.5-sonnet Scores', {}) scores_3c3h = scores_data.get('3C3H Scores', {}) scores_tasks = scores_data.get('Tasks Scores', {}) # Multiply scores by 100 to get percentages (keep them as numeric values) formatted_scores_3c3h = {k: v*100 for k, v in scores_3c3h.items()} formatted_scores_tasks = {k: v*100 for k, v in scores_tasks.items()} # For 3C3H Scores DataFrame data_entry_3c3h = { 'Model Name': model_name, 'Revision': revision, 'License': license, 'Precision': precision, 'Model Size': model_size_numeric, # Numeric value for sorting '3C3H Score': formatted_scores_3c3h.get("3C3H Score", np.nan), 'Correctness': formatted_scores_3c3h.get("Correctness", np.nan), 'Completeness': formatted_scores_3c3h.get("Completeness", np.nan), 'Conciseness': formatted_scores_3c3h.get("Conciseness", np.nan), 'Helpfulness': formatted_scores_3c3h.get("Helpfulness", np.nan), 'Honesty': formatted_scores_3c3h.get("Honesty", np.nan), 'Harmlessness': formatted_scores_3c3h.get("Harmlessness", np.nan), } data_3c3h.append(data_entry_3c3h) # For Tasks Scores DataFrame data_entry_tasks = { 'Model Name': model_name, 'Revision': revision, 'License': license, 'Precision': precision, 'Model Size': model_size_numeric, # Numeric value for sorting **formatted_scores_tasks } data_tasks.append(data_entry_tasks) df_3c3h = pd.DataFrame(data_3c3h) df_tasks = pd.DataFrame(data_tasks) # Round the numeric score columns to 4 decimal places score_columns_3c3h = ['3C3H Score', 'Correctness', 'Completeness', 'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness'] df_3c3h[score_columns_3c3h] = df_3c3h[score_columns_3c3h].round(4) # Replace np.inf with a large number in 'Model Size Filter' for filtering max_model_size_value = 1000 # Define a maximum value df_3c3h['Model Size Filter'] = df_3c3h['Model Size'].replace(np.inf, max_model_size_value) # Sort df_3c3h by '3C3H Score' descending if column exists if '3C3H Score' in df_3c3h.columns: df_3c3h = df_3c3h.sort_values(by='3C3H Score', ascending=False) df_3c3h.insert(0, 'Rank', range(1, len(df_3c3h) + 1)) # Add Rank column starting from 1 else: df_3c3h.insert(0, 'Rank', range(1, len(df_3c3h) + 1)) # Extract task columns task_columns = [col for col in df_tasks.columns if col not in ['Model Name', 'Revision', 'License', 'Precision', 'Model Size', 'Model Size Filter']] # Round the task score columns to 4 decimal places if task_columns: df_tasks[task_columns] = df_tasks[task_columns].round(4) # Replace np.inf with a large number in 'Model Size Filter' for filtering df_tasks['Model Size Filter'] = df_tasks['Model Size'].replace(np.inf, max_model_size_value) # Sort df_tasks by the first task column if it exists if task_columns: first_task = task_columns[0] df_tasks = df_tasks.sort_values(by=first_task, ascending=False) df_tasks.insert(0, 'Rank', range(1, len(df_tasks) + 1)) # Add Rank column starting from 1 else: df_tasks = df_tasks.sort_values(by='Model Name', ascending=True) df_tasks.insert(0, 'Rank', range(1, len(df_tasks) + 1)) return df_3c3h, df_tasks, task_columns def load_requests(status_folder): api = HfApi() requests_data = [] folder_path_in_repo = status_folder # 'pending', 'finished', or 'failed' hf_api_token = os.environ.get('HF_API_TOKEN', None) try: # List files in the dataset repository files_info = api.list_repo_files( repo_id=DATASET_REPO_ID, repo_type="dataset", token=hf_api_token ) except Exception as e: print(f"Error accessing dataset repository: {e}") return pd.DataFrame() # Return empty DataFrame if repository not found or inaccessible # Filter files in the desired folder files_in_folder = [f for f in files_info if f.startswith(f"{folder_path_in_repo}/") and f.endswith('.json')] for file_path in files_in_folder: try: # Download the JSON file local_file_path = hf_hub_download( repo_id=DATASET_REPO_ID, filename=file_path, repo_type="dataset", token=hf_api_token ) # Load JSON data with open(local_file_path, 'r') as f: request = json.load(f) requests_data.append(request) except Exception as e: print(f"Error loading file {file_path}: {e}") continue # Skip files that can't be loaded df = pd.DataFrame(requests_data) return df def submit_model(model_name, revision, precision, params, license): # Load existing evaluations df_3c3h, df_tasks, _ = load_results() existing_models_results = df_3c3h[['Model Name', 'Revision', 'Precision']] # Handle 'Missing' precision if precision == 'Missing': precision = None else: precision = precision.strip().lower() # Load pending and finished requests from the dataset repository df_pending = load_requests('pending') df_finished = load_requests('finished') # Check if model is already evaluated model_exists_in_results = ((existing_models_results['Model Name'] == model_name) & (existing_models_results['Revision'] == revision) & (existing_models_results['Precision'] == precision)).any() if model_exists_in_results: return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**" # Check if model is in pending requests if not df_pending.empty: existing_models_pending = df_pending[['model_name', 'revision', 'precision']] model_exists_in_pending = ((existing_models_pending['model_name'] == model_name) & (existing_models_pending['revision'] == revision) & (existing_models_pending['precision'] == precision)).any() if model_exists_in_pending: return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' is already in the pending evaluations.**" # Check if model is in finished requests if not df_finished.empty: existing_models_finished = df_finished[['model_name', 'revision', 'precision']] model_exists_in_finished = ((existing_models_finished['model_name'] == model_name) & (existing_models_finished['revision'] == revision) & (existing_models_finished['precision'] == precision)).any() if model_exists_in_finished: return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**" # Check if model exists on HuggingFace Hub api = HfApi() try: model_info = api.model_info(model_name) except Exception as e: return f"**Error: Could not find model '{model_name}' on HuggingFace Hub. Please ensure the model name is correct and the model is public.**" # Proceed with submission status = "PENDING" # Prepare the submission data submission = { "model_name": model_name, "license": license, "revision": revision, "precision": precision, "status": status, "params": params } # Serialize the submission to JSON submission_json = json.dumps(submission, indent=2) # Define the file path in the repository org_model = model_name.split('/') if len(org_model) != 2: return "**Please enter the full model name including the organization or username, e.g., 'inceptionai/jais-family-30b-8k'**" org, model_id = org_model precision_str = precision if precision else 'Missing' file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}.json" # Upload the submission to the dataset repository try: hf_api_token = os.environ.get('HF_API_TOKEN', None) api.upload_file( path_or_fileobj=submission_json.encode('utf-8'), path_in_repo=file_path_in_repo, repo_id=DATASET_REPO_ID, repo_type="dataset", token=hf_api_token ) except Exception as e: return f"**Error: Could not submit the model. {str(e)}**" return f"**Model '{model_name}' has been submitted for evaluation.**" def main(): df_3c3h, df_tasks, task_columns = load_results() # Extract unique Precision and License values for filters precision_options_3c3h = sorted(df_3c3h['Precision'].dropna().unique().tolist()) precision_options_3c3h = [p for p in precision_options_3c3h if p != 'UNK'] precision_options_3c3h.append('Missing') license_options_3c3h = sorted(df_3c3h['License'].dropna().unique().tolist()) license_options_3c3h = [l for l in license_options_3c3h if l != 'UNK'] license_options_3c3h.append('Missing') precision_options_tasks = sorted(df_tasks['Precision'].dropna().unique().tolist()) precision_options_tasks = [p for p in precision_options_tasks if p != 'UNK'] precision_options_tasks.append('Missing') license_options_tasks = sorted(df_tasks['License'].dropna().unique().tolist()) license_options_tasks = [l for l in license_options_tasks if l != 'UNK'] license_options_tasks.append('Missing') # Get min and max model sizes for sliders, handling 'inf' values min_model_size_3c3h = int(df_3c3h['Model Size Filter'].min()) max_model_size_3c3h = int(df_3c3h['Model Size Filter'].max()) min_model_size_tasks = int(df_tasks['Model Size Filter'].min()) max_model_size_tasks = int(df_tasks['Model Size Filter'].max()) # Exclude 'Model Size Filter' from column selectors column_choices_3c3h = [col for col in df_3c3h.columns if col != 'Model Size Filter'] column_choices_tasks = [col for col in df_tasks.columns if col != 'Model Size Filter'] with gr.Blocks() as demo: gr.HTML(HEADER) with gr.Tabs(): with gr.Tab("Leaderboard"): with gr.Tabs(): with gr.Tab("3C3H Scores"): with gr.Row(): search_box_3c3h = gr.Textbox( placeholder="Search for models...", label="Search", interactive=True ) with gr.Row(): column_selector_3c3h = gr.CheckboxGroup( choices=column_choices_3c3h, value=[ 'Rank', 'Model Name', '3C3H Score', 'Correctness', 'Completeness', 'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness' ], label="Select columns to display", ) with gr.Row(): license_filter_3c3h = gr.CheckboxGroup( choices=license_options_3c3h, value=license_options_3c3h.copy(), # Default all selected label="Filter by License", ) precision_filter_3c3h = gr.CheckboxGroup( choices=precision_options_3c3h, value=precision_options_3c3h.copy(), # Default all selected label="Filter by Precision", ) with gr.Row(): model_size_min_filter_3c3h = gr.Slider( minimum=min_model_size_3c3h, maximum=max_model_size_3c3h, value=min_model_size_3c3h, step=1, label="Minimum Model Size", interactive=True ) model_size_max_filter_3c3h = gr.Slider( minimum=min_model_size_3c3h, maximum=max_model_size_3c3h, value=max_model_size_3c3h, step=1, label="Maximum Model Size", interactive=True ) leaderboard_3c3h = gr.Dataframe( df_3c3h[['Rank', 'Model Name', '3C3H Score', 'Correctness', 'Completeness', 'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness']], interactive=False ) def filter_df_3c3h(search_query, selected_cols, precision_filters, license_filters, min_size, max_size): filtered_df = df_3c3h.copy() # Ensure min_size <= max_size if min_size > max_size: min_size, max_size = max_size, min_size # Apply search filter if search_query: filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)] # Apply Precision filter if precision_filters: include_missing = 'Missing' in precision_filters selected_precisions = [p for p in precision_filters if p != 'Missing'] if include_missing: filtered_df = filtered_df[ (filtered_df['Precision'].isin(selected_precisions)) | (filtered_df['Precision'] == 'UNK') | (filtered_df['Precision'].isna()) ] else: filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)] # Apply License filter if license_filters: include_missing = 'Missing' in license_filters selected_licenses = [l for l in license_filters if l != 'Missing'] if include_missing: filtered_df = filtered_df[ (filtered_df['License'].isin(selected_licenses)) | (filtered_df['License'] == 'UNK') | (filtered_df['License'].isna()) ] else: filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)] # Apply Model Size filter filtered_df = filtered_df[ (filtered_df['Model Size Filter'] >= min_size) & (filtered_df['Model Size Filter'] <= max_size) ] # Remove existing 'Rank' column if present if 'Rank' in filtered_df.columns: filtered_df = filtered_df.drop(columns=['Rank']) # Recalculate Rank after filtering filtered_df = filtered_df.reset_index(drop=True) filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1)) # Ensure selected columns are present selected_cols = [col for col in selected_cols if col in filtered_df.columns] return filtered_df[selected_cols] # Bind the filter function to the appropriate events filter_inputs_3c3h = [ search_box_3c3h, column_selector_3c3h, precision_filter_3c3h, license_filter_3c3h, model_size_min_filter_3c3h, model_size_max_filter_3c3h ] search_box_3c3h.submit( filter_df_3c3h, inputs=filter_inputs_3c3h, outputs=leaderboard_3c3h ) # Bind change events for CheckboxGroups and sliders for component in filter_inputs_3c3h: component.change( filter_df_3c3h, inputs=filter_inputs_3c3h, outputs=leaderboard_3c3h ) with gr.Tab("Tasks Scores"): gr.Markdown(""" Note: This Table is sorted based on the First Task (Question Answering) """) with gr.Row(): search_box_tasks = gr.Textbox( placeholder="Search for models...", label="Search", interactive=True ) with gr.Row(): column_selector_tasks = gr.CheckboxGroup( choices=column_choices_tasks, value=['Rank', 'Model Name'] + task_columns, label="Select columns to display", ) with gr.Row(): license_filter_tasks = gr.CheckboxGroup( choices=license_options_tasks, value=license_options_tasks.copy(), # Default all selected label="Filter by License", ) precision_filter_tasks = gr.CheckboxGroup( choices=precision_options_tasks, value=precision_options_tasks.copy(), # Default all selected label="Filter by Precision", ) with gr.Row(): model_size_min_filter_tasks = gr.Slider( minimum=min_model_size_tasks, maximum=max_model_size_tasks, value=min_model_size_tasks, step=1, label="Minimum Model Size", interactive=True ) model_size_max_filter_tasks = gr.Slider( minimum=min_model_size_tasks, maximum=max_model_size_tasks, value=max_model_size_tasks, step=1, label="Maximum Model Size", interactive=True ) leaderboard_tasks = gr.Dataframe( df_tasks[['Rank', 'Model Name'] + task_columns], interactive=False ) def filter_df_tasks(search_query, selected_cols, precision_filters, license_filters, min_size, max_size): filtered_df = df_tasks.copy() # Ensure min_size <= max_size if min_size > max_size: min_size, max_size = max_size, min_size # Apply search filter if search_query: filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)] # Apply Precision filter if precision_filters: include_missing = 'Missing' in precision_filters selected_precisions = [p for p in precision_filters if p != 'Missing'] if include_missing: filtered_df = filtered_df[ (filtered_df['Precision'].isin(selected_precisions)) | (filtered_df['Precision'] == 'UNK') | (filtered_df['Precision'].isna()) ] else: filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)] # Apply License filter if license_filters: include_missing = 'Missing' in license_filters selected_licenses = [l for l in license_filters if l != 'Missing'] if include_missing: filtered_df = filtered_df[ (filtered_df['License'].isin(selected_licenses)) | (filtered_df['License'] == 'UNK') | (filtered_df['License'].isna()) ] else: filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)] # Apply Model Size filter filtered_df = filtered_df[ (filtered_df['Model Size Filter'] >= min_size) & (filtered_df['Model Size Filter'] <= max_size) ] # Remove existing 'Rank' column if present if 'Rank' in filtered_df.columns: filtered_df = filtered_df.drop(columns=['Rank']) # Sort by the first task column if it exists if task_columns: first_task = task_columns[0] filtered_df = filtered_df.sort_values(by=first_task, ascending=False) else: filtered_df = filtered_df.sort_values(by='Model Name', ascending=True) # Recalculate Rank after filtering filtered_df = filtered_df.reset_index(drop=True) filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1)) # Ensure selected columns are present selected_cols = [col for col in selected_cols if col in filtered_df.columns] return filtered_df[selected_cols] # Bind the filter function to the appropriate events filter_inputs_tasks = [ search_box_tasks, column_selector_tasks, precision_filter_tasks, license_filter_tasks, model_size_min_filter_tasks, model_size_max_filter_tasks ] search_box_tasks.submit( filter_df_tasks, inputs=filter_inputs_tasks, outputs=leaderboard_tasks ) # Bind change events for CheckboxGroups and sliders for component in filter_inputs_tasks: component.change( filter_df_tasks, inputs=filter_inputs_tasks, outputs=leaderboard_tasks ) with gr.Tab("Submit Here"): gr.Markdown(ABOUT_SECTION) gr.Markdown("---") gr.Markdown("# Submit Your Model for Evaluation") with gr.Column(): model_name_input = gr.Textbox( label="Model Name", placeholder="Enter the full model name from HuggingFace Hub (e.g., inceptionai/jais-family-30b-8k)" ) revision_input = gr.Textbox( label="Revision", placeholder="main", value="main" ) precision_input = gr.Dropdown( choices=["float16", "float32", "bfloat16", "8bit", "4bit"], label="Precision", value="float16" ) params_input = gr.Textbox( label="Params", placeholder="Enter the approximate number of parameters as Integer (e.g., 7, 13, 30, 70 ...)" ) # Changed from Dropdown to Textbox with default value "Open" license_input = gr.Textbox( label="License", placeholder="Enter the license type (Generic one is 'Open' in case no License is provided)", value="Open" ) submit_button = gr.Button("Submit Model") submission_result = gr.Markdown() submit_button.click( submit_model, inputs=[model_name_input, revision_input, precision_input, params_input, license_input], outputs=submission_result ) # Load pending, finished, and failed requests df_pending = load_requests('pending') df_finished = load_requests('finished') df_failed = load_requests('failed') # Display the tables gr.Markdown("## Evaluation Status") with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False): if not df_pending.empty: gr.Dataframe(df_pending) else: gr.Markdown("No pending evaluations.") with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False): if not df_finished.empty: gr.Dataframe(df_finished) else: gr.Markdown("No finished evaluations.") with gr.Accordion(f"Failed Evaluations ({len(df_failed)})", open=False): if not df_failed.empty: gr.Dataframe(df_failed) else: gr.Markdown("No failed evaluations.") with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True, ) demo.launch() if __name__ == "__main__": main()