create leaderboard

app.py

@@ -0,0 +1,181 @@
+import gradio as gr
+import pandas as pd
+import random
+import time 
+
+from info.train_a_model import (
+    LLM_BENCHMARKS_TEXT)
+from info.submit import (
+    SUBMIT_TEXT)
+from info.deployment import (
+    DEPLOY_TEXT)
+from info.programs import (
+    PROGRAMS_TEXT)
+from info.citation import(
+    CITATION_TEXT)
+from src.processing import filter_benchmarks_table, make_clickable
+
+demo = gr.Blocks()
+
+with demo:
+    
+    gr.HTML("""<h1 align="center" id="space-title">🤗Powered-by-Intel LLM Leaderboard 💻</h1>""")
+    gr.Markdown("This leaderboard is designed to evaluate, score, and rank open-source large language \
+        models that have been pre-trained or fine-tuned on Intel Hardware 🦾")
+    gr.Markdown("Models submitted to the leaderboard are evaluated \
+        on the Intel Developer Cloud ☁️")
+    
+    # TODO: Coming soon comparison tool
+    #with gr.Accordion("🥊Large Language Model Boxing Ring 🥊", open=False):
+    #    with gr.Row():
+    #        chat_a = gr.Chatbot()
+    #        chat_b = gr.Chatbot()
+    #    msg = gr.Textbox()
+    #    gr.ClearButton([msg, chat_a])
+#
+    #    def respond(message, chat_history):
+    #        bot_message = random.choice(["How are you?", "I love you", "I'm very hungry"])
+    #        chat_history.append((message, bot_message))
+    #        time.sleep(2)
+    #        return "", chat_history
+#
+    #    msg.submit(respond, inputs = [msg, chat_a],outputs = [msg, chat_a]) 
+        
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏆 LLM Benchmark", elem_id="llm-benchmark-table", id=0):
+            with gr.Row():
+                with gr.Column():
+                    filter_hw = gr.CheckboxGroup(choices=["Gaudi","Xeon","GPU Max","Arc GPU","Core Ultra"],
+                                     label="Select Training Platform*",
+                                     elem_id="compute_platforms",
+                                     value=["Gaudi","Xeon","GPU Max","Arc GPU","Core Ultra"])
+                    filter_platform = gr.CheckboxGroup(choices=["Intel Developer Cloud","AWS","Azure","GCP","Local"],
+                                     label="Training Infrastructure*",
+                                     elem_id="training_infra",
+                                     value=["Intel Developer Cloud","AWS","Azure","GCP","Local"])
+                    filter_affiliation = gr.CheckboxGroup(choices=["No Affiliation","Intel Innovator","Intel Student Ambassador", "Intel Software Liftoff", "Intel Labs", "Other"],
+                                     label="Intel Program Affiliation",
+                                     elem_id="program_affiliation",
+                                     value=["No Affiliation","Intel Innovator","Intel Student Ambassador", "Intel Software Liftoff", "Intel Labs", "Other"])
+                    
+                with gr.Column():
+                    filter_size = gr.CheckboxGroup(choices=[1,3,5,7,13,35,60,70,100],
+                                     label="Model Sizes (Billion of Parameters)",
+                                     elem_id="parameter_size",
+                                     value=[1,3,5,7,13,35,60,70,100])
+                    filter_precision = gr.CheckboxGroup(choices=["fp8","fp16","bf16","int8","4bit"],
+                                     label="Model Precision",
+                                     elem_id="precision",
+                                     value=["fp8","fp16","bf16","int8","4bit"])
+                    filter_type = gr.CheckboxGroup(choices=["pretrained","fine-tuned","chat-models","merges/moerges"],
+                                     label="Model Types",
+                                     elem_id="model_types",
+                                     value=["pretrained","fine-tuned","chat-models","merges/moerges"])
+                    
+            initial_df = pd.read_csv("leaderboard_status_030424.csv")
+            
+            gradio_df_display = gr.Dataframe()
+            
+            def update_df(hw_selected, platform_selected, affiliation_selected, size_selected, precision_selected, type_selected):
+                filtered_df = filter_benchmarks_table(df=initial_df, hw_selected=hw_selected, platform_selected=platform_selected, 
+                                                      affiliation_selected=affiliation_selected, size_selected=size_selected, 
+                                                      precision_selected=precision_selected, type_selected=type_selected)
+                return filtered_df
+            
+            filter_hw.change(fn=update_df, 
+                             inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
+                             outputs=[gradio_df_display])
+            filter_platform.change(fn=update_df, 
+                                inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
+                                outputs=[gradio_df_display])
+            filter_affiliation.change(fn=update_df, 
+                                inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
+                                outputs=[gradio_df_display])
+            filter_size.change(fn=update_df, 
+                               inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
+                               outputs=[gradio_df_display])
+            filter_precision.change(fn=update_df, 
+                                inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
+                                outputs=[gradio_df_display])
+            filter_type.change(fn=update_df, 
+                               inputs=[filter_hw, filter_platform, filter_affiliation, filter_size, filter_precision, filter_type], 
+                               outputs=[gradio_df_display])
+        
+            
+        with gr.TabItem("🧰 Train a Model", elem_id="getting-started", id=1):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("🚀 Deployment Tips", elem_id="deployment-tips", id=2):
+            gr.Markdown(DEPLOY_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("👩‍💻 Developer Programs", elem_id="hardward-program", id=3):
+            gr.Markdown(PROGRAMS_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("🏎️ Submit", elem_id="submit", id=4):
+            gr.Markdown(SUBMIT_TEXT, elem_classes="markdown-text")
+            with gr.Row():
+                gr.Markdown("# Submit Model for Evaluation 🏎️", elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    model_type = gr.Dropdown(
+                        choices=["pretrained","fine-tuned","chat models","merges/moerges"],
+                        label="Model type",
+                        multiselect=False,
+                        value="pretrained",
+                        interactive=True,
+                    )
+                    
+                    hw_type = gr.Dropdown(
+                        choices=["Gaudi","Xeon","GPU Max","Arc GPU"],
+                        label="Training Hardware",
+                        multiselect=False,
+                        value="Gaudi2",
+                        interactive=True,
+                    )
+                    terms = gr.Checkbox(
+                        label="Check if you have read and agreed to terms and conditions associated with submitting\
+                            a model to the leaderboard.",
+                        value=False,
+                        interactive=True,
+                    )
+                with gr.Column():
+                    precision = gr.Dropdown(
+                        choices=["fp8","fp16","bf16","int8","4bit"],
+                        label="Precision",
+                        multiselect=False,
+                        value="fp16",
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=["Original", "Adapter", "Delta"],
+                        label="Weights type",
+                        multiselect=False,
+                        value="Original",
+                        interactive=True,
+                    )
+                    training_infra = gr.Dropdown(
+                        choices=["IDC","AWS","Azure","GCP","Local"],
+                        label="Training Infrastructure",
+                        multiselect=False,
+                        value="IDC",
+                        interactive=True,
+                    )
+                    affiliation = gr.Dropdown(
+                        choices=["No Affiliation","Innovator","Student Ambassador","Intel Liftoff", "Intel Labs", "Other"],
+                        label="Affiliation with Intel",
+                        multiselect=False,
+                        value="Independent",
+                        interactive=True,
+                    )
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                
+            #submit_button = gr.Button("Submit Eval")
+            #submission_result = gr.Markdown()
+            gr.Markdown("Community Submissions Coming soon!")
+            
+    with gr.Accordion("📙 Citation", open=False):
+            citation =gr.Textbox(value = CITATION_TEXT,
+                                 lines=6,
+                                 label="Use the following to cite this content")
+            
+    
+demo.launch()

info/citation.py

@@ -0,0 +1,8 @@
+CITATION_TEXT = r"""@misc{powered-by-intel-llm-leaderboard,
+  author = {Eduardo Alvarez},
+  title = {Powered By Intel LLM Leaderboard},
+  year = {2024},
+  publisher = {Intel},
+  howpublished = "\url{https://huggingface.co/spaces/Intel/powered_by_intel_leaderboard}"
+}
+"""

info/deployment.py

@@ -0,0 +1,126 @@
+DEPLOY_TEXT = f"""
+
+Having table full of powerful models is nice and call but at the end of the day, you have to be able to use
+them for something. Below you will find sample code to help you load models and perform inference. 
+
+
+## Inference with Gaudi 2
+Habana's SDK, Intel Gaudi Software, supports PyTorch and DeepSpeed for accelerating LLM training and inference. 
+The Intel Gaudi Software graph compiler will optimize the execution of the operations accumulated in the graph 
+(e.g. operator fusion, data layout management, parallelization, pipelining and memory management, 
+and graph-level optimizations).
+
+Optimum Habana provides covenient functionality for various tasks, below you'll find the command line
+snippet that you would run to perform inference on Gaudi with meta-llama/Llama-2-7b-hf.
+
+The "run_generation.py" script below can be found [here](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)
+
+```bash
+python run_generation.py \
+--model_name_or_path meta-llama/Llama-2-7b-hf \
+--use_hpu_graphs \
+--use_kv_cache \
+--max_new_tokens 100 \
+--do_sample \
+--batch_size 2 \
+--prompt "Hello world" "How are you?"
+
+```
+
+# Inference Intel Extension for Transformers
+Intel® Extension for Transformers is an innovative toolkit designed to accelerate GenAI/LLM 
+everywhere with the optimal performance of Transformer-based models on various Intel platforms, 
+including Intel Gaudi2, Intel CPU, and Intel GPU. 
+
+### INT4 Inference (CPU)
+```python
+from transformers import AutoTokenizer
+from intel_extension_for_transformers.transformers import AutoModelForCausalLM
+model_name = "Intel/neural-chat-7b-v3-1"     
+prompt = "When winter becomes spring, the flowers..."
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+inputs = tokenizer(prompt, return_tensors="pt").input_ids
+
+model = AutoModelForCausalLM.from_pretrained(model_name, load_in_4bit=True)
+outputs = model.generate(inputs)
+
+```
+### INT4 Inference (GPU)
+```python
+import intel_extension_for_pytorch as ipex
+from intel_extension_for_transformers.transformers.modeling import AutoModelForCausalLM
+from transformers import AutoTokenizer
+
+device_map = "xpu"
+model_name ="Qwen/Qwen-7B"
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+prompt = "When winter becomes spring, the flowers..."
+inputs = tokenizer(prompt, return_tensors="pt").input_ids.to(device_map)
+
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True,
+                                              device_map=device_map, load_in_4bit=True)
+
+model = ipex.optimize_transformers(model, inplace=True, dtype=torch.float16, woq=True, device=device_map)
+
+output = model.generate(inputs)
+```
+
+# Intel Extension for PyTorch
+Intel® Extension for PyTorch extends PyTorch with up-to-date features optimizations for an 
+extra performance boost on Intel hardware. Optimizations take advantage of Intel® Advanced 
+Vector Extensions 512 (Intel® AVX-512) Vector Neural Network Instructions (VNNI) and Intel® 
+Advanced Matrix Extensions (Intel® AMX) on Intel CPUs as well as Intel Xe Matrix Extensions 
+(XMX) AI engines on Intel discrete GPUs. Moreover, Intel® Extension for PyTorch* provides easy 
+GPU acceleration for Intel discrete GPUs through the PyTorch* xpu device.
+
+There are a few flavors of PyTorch that can be leveraged for inference. For detailed documentation,
+the visit https://intel.github.io/intel-extension-for-pytorch/#introduction
+
+### IPEX with Optimum Intel (no quantization)
+Requires installing/updating optimum `pip install --upgrade-strategy eager optimum[ipex]
+`
+```python
+from optimum.intel import IPEXModelForCausalLM
+from transformers import AutoTokenizer, pipeline
+
+model = IPEXModelForCausalLM.from_pretrained(model_id)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+results = pipe("A fisherman at sea...")
+```
+
+### IPEX with Stock PyTorch with Mixed Precision
+```python
+import torch
+import intel_extension_for_pytorch as ipex
+import transformers
+
+model= transformers.AutoModelForCausalLM(model_name_or_path).eval()
+
+dtype = torch.float # or torch.bfloat16
+model = ipex.llm.optimize(model, dtype=dtype)
+
+# generation inference loop
+with torch.inference_mode():
+    model.generate()
+```
+
+# OpenVINO Toolkit
+
+```python
+from optimum.intel import OVModelForCausalLM
+from transformers import AutoTokenizer, pipeline
+
+model_id = "helenai/gpt2-ov"
+model = OVModelForCausalLM.from_pretrained(model_id)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+
+pipe("In the spring, beautiful flowers bloom...")
+
+```
+
+
+"""

info/programs.py

@@ -0,0 +1,37 @@
+PROGRAMS_TEXT= """
+Intel offers a range of programs to grant early, short, and long-term access to developers. A great way to build
+and share models on the "Powered by Intel" LLM Leaderboard is to join one of these programs. Learn more about
+these opportunities below: 
+
+## Intel Liftoff Program
+Intel® Liftoff for startups is open to early-stage AI and machine learning startups. This free virtual program 
+helps you innovate and scale, no matter where you are in your entrepreneurial journey.
+
+Through Intel Liftoff, startups can access the computational power they need to build powerful LLMs on platforms
+like Gaudi, Max Series GPUs, and Xeon Processors.
+
+Learn more and apply through the program at https://www.intel.com/content/www/us/en/developer/tools/oneapi/liftoff.html
+
+## Intel Student Ambassador Program
+This program is focused on undergraduate and graduate students who are passionate about technology and 
+working with developer communities to promote learning, sharing, and collaboration. It provides opportunities 
+for students to enhance their AI and oneAPI skills, expand their network, and learn about the cutting-edge Intel® 
+hardware and software products.
+
+Through the Student Ambassador Program you can get early access to latest technology developments that are under 
+a nondisclosure agreement (NDA) and extended access to Intel® Developer Cloud.
+
+Learn more and apply through the program at https://www.intel.com/content/www/us/en/developer/tools/oneapi/training/academic-program/student-ambassador.html#gs.5f5oi3
+
+
+## Intel Innovator Program
+This program is for developers using oneAPI on Intel® architecture who provide technical leadership and inspiration 
+to the global developer community through their projects, expertise, and advocacies. It provides recognition as a 
+oneAPI expert, opportunities for event support, spotlight showcase for projects, and more.
+
+Through this program you will have free ccess to paid features on Intel® Developer Cloud, a development 
+environment for projects on the latest Intel technology and as a oneAPI expert, and interactions with 
+others in the community​ and within Intel
+
+Learn more and apply through the program at https://www.intel.com/content/www/us/en/developer/community/innovators/oneapi-innovator.html
+"""

info/submit.py

@@ -0,0 +1,37 @@
+
+SUBMIT_TEXT = f"""
+# Evaluation Queue for the 🤗"Powered by Intel" LLM Leaderboard 💻
+Models added here will be queued for evaluation on the Intel Developer Cloud ☁️
+## First steps before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+We use your model card to better understand the properties of your model and make them more easily discoverable for other users. 
+Model cards are required to have mentions of the hardware, software, and infrastructure used for training - without this information 
+we cannot accept your model as a valid submission. 
+
+### 5) Select the correct precision
+Not all models are converted properly from `float16` to `bfloat16`, and selecting the wrong precision can sometimes cause evaluation error (as loading a `bf16` model in `fp16` can sometimes generate NaNs, depending on the weight range).
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the command in the About tab under "Reproducibility" with all arguments specified (you can add `--limit` to limit the number of examples per task).
+
+"""

info/train_a_model.py

@@ -0,0 +1,73 @@
+
+LLM_BENCHMARKS_TEXT = f"""
+# Use the Resources Below to Start Training a Model Today
+
+Intel offers a variety of platforms that can be used to train LLMs including datacenter and consumer grade cpus, gpus, and ASICs. 
+Below, you'll find documentation on how to access free and paid resources to train a model and submit it to the "Intel Inside Leaderboard". 
+
+## Intel Developer Cloud - Quick Start
+The Intel Developer Cloud is one of the best places to access free and paid compute instances for model training. Intel offers Jupyter Notebook instances supported by
+224 Core 4th Generation Xeon Baremetal nodes with 4x Max Series GPU 1100 GPUs. To access these resources please follow the instructions below:
+1. Visit [cloud.intel.com](cloud.intel.com) and create a free account. 
+2. Navigate to the "Training" module under the "Software" section in the left panel
+3. Under the GenAI Essentials section, select the LLM Fine-Tuning with QLoRA notebook and click "Launch"
+4. Follow the instructions in the notebook to train your model using Intel Max Series 1100 GPUs
+5. Upload your model to the Hugging Face Model Hub
+6. Go to the "Submit" tab follow instructions to create a leaderboard evaluation request
+
+## Additional Training Code Samples
+
+Below you will find a list of additional resources for training models on different intel hardware platforms:
+- Gaudi Processors
+    - [Parameter Efficient Fine-Tuning of Llama-2 70B](https://github.com/HabanaAI/Gaudi-tutorials/blob/main/PyTorch/llama2_fine_tuning_inference/llama2_fine_tuning_inference.ipynb)
+- Xeon Processors
+    - [Distributed Training of GPT2 LLMs on AWS](https://github.com/intel/intel-cloud-optimizations-aws/tree/main/distributed-training)
+    - [Fine-tuning Falcon 7B on Xeon Processors](https://medium.com/@eduand-alvarez/fine-tune-falcon-7-billion-on-xeon-cpus-with-hugging-face-and-oneapi-a25e10803a53)
+- Max Series GPUs
+    - [LLM Fine-tuning with QLoRA on Max Series GPUs](https://console.idcservice.net/training/detail/159c24e4-5598-3155-a790-2qv973tlm172)
+## Submitting your Model to the Hub
+Once you have trained your model, it is a straighforward process to upload and open source it on the Hugging Face Hub. 
+
+```python
+
+# Logging in to Hugging Face
+
+from huggingface_hub import notebook_login, Repository
+
+# Login to Hugging Face
+notebook_login()
+
+# Model and Tokenize Loading
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+# Define the path to the checkpoint
+checkpoint_path = ""  # Replace with your checkpoint folder
+
+# Load the model
+model = AutoModelForSequenceClassification.from_pretrained(checkpoint_path)
+
+# Load the tokenizer
+tokenizer = AutoTokenizer.from_pretrained("") #add name of your model's tokenizer on Hugging Face OR custom tokenizer
+
+#Saving and Uploading the Model and Tokenizer
+
+# Save the model and tokenizer
+model_name_on_hub = "desired-model-name"
+model.save_pretrained(model_name_on_hub)
+tokenizer.save_pretrained(model_name_on_hub)
+
+# Push to the hub
+model.push_to_hub(model_name_on_hub)
+tokenizer.push_to_hub(model_name_on_hub)
+
+# Congratulations! Your fine-tuned model is now uploaded to the Hugging Face Model Hub. 
+# You can view and share your model using its URL: https://huggingface.co/your-username/your-model-name
+
+```
+
+"""
+
+SUBMIT_TEXT = f"""
+# Use the Resource Below to Start Training a Model Today
+
+"""

leaderboard_status_030424.csv

@@ -0,0 +1,5 @@
+Model,Average,Hardware,Model Type,Precision,Size,Infrastructure,ARC,HellaSwag,MMLU,TruthfulQA,Winogrande,GSM8K,Affiliation
+Intel/neural-chat-7b-v3-3,69.83,Gaudi,fine-tuned,fp16,7,Intel Developer Cloud,66.89,85.26,63.07,63.01,79.64,61.11,Intel Labs
+Intel/neural-chat-7b-v3-2,68.29,Gaudi,fine-tuned,fp16,7,Intel Developer Cloud,67.49,83.92,63.55,59.68,79.65,55.12,Intel Labs
+Intel/neural-chat-7b-v3-1,61.59,Gaudi,fine-tuned,fp16,7,Intel Developer Cloud,66.21,83.64,62.37,59.65,78.14,19.56,Intel Labs
+Intel/neural-chat-7b-v3,58.46,Gaudi,fine-tuned,fp16,7,Intel Developer Cloud,67.15,83.29,62.26,58.77,78.06,1.21,Intel Labs

src/leaderboard_filtered.csv

@@ -0,0 +1,7 @@
+,Model,Average,Hardware,Model Type,Precision,Size,Infrastructure,ARC,MMLU,TruthfulQA,Winogrande,GSM8K,Affiliation
+1,BetaWave,83.21,Arc GPU,fine-tuned,fp16,7,Local,70.44,92.32,78.67,85.55,90.0,Innovator
+4,EpsilonWave,58.44,Xeon,fine-tuned,int8,3,AWS,91.22,82.1,60.55,80.11,77.89,Partner
+6,EtaMatrix,69.78,Xeon,fine-tuned,int8,3,GCP,85.55,79.33,70.89,72.18,79.44,Liftoff
+7,ThetaCore,88.12,Arc GPU,fine-tuned,int8,3,Local,67.33,85.78,88.55,86.9,83.11,Liftoff
+14,BetaNeural,79.67,Gaudi 1,fine-tuned,4bit,7,AWS,85.44,77.22,83.1,75.45,71.33,Partner
+15,TrackSpeed,88.12,Arc GPU,fine-tuned,4bit,7,Local,67.33,85.78,88.55,86.9,83.11,Student Ambassador

src/processing.py

@@ -0,0 +1,20 @@
+
+def filter_benchmarks_table(df, hw_selected, platform_selected,
+                            size_selected, precision_selected,
+                            type_selected, affiliation_selected):
+ 
+    filtered_df = df[
+    df['Hardware'].isin(hw_selected) &
+    df['Infrastructure'].isin(platform_selected) &
+    df['Size'].isin(size_selected) &
+    df['Precision'].isin(precision_selected) &
+    df['Model Type'].isin(type_selected) &
+    df['Affiliation'].isin(affiliation_selected)]
+
+    return filtered_df
+
+def make_clickable(val):
+    return f'<a target="_blank" href="{val}">{val}</a>'
+
+
+