Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| from evaluate import evaluator | |
| import evaluate | |
| import datasets | |
| from huggingface_hub import HfApi, ModelFilter | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification | |
| from transformers import AutoTokenizer, AutoModelForMaskedLM | |
| from transformers import pipeline, AutoConfig | |
| import matplotlib.pyplot as plt | |
| st.title("Metric Compare") | |
| st.markdown("### Choose the dataset you want to use for the comparison:") | |
| api = HfApi() | |
| dsets = [d.id for d in api.list_datasets(filter="task_categories:text-classification", sort = "downloads", direction=-1, limit = 20) if d.id !='glue'] | |
| dset = st.selectbox('Choose a dataset from the Hub', options=dsets) | |
| info = datasets.get_dataset_infos(dset) | |
| dset_config = st.selectbox('What config do you want to use?', options=list(info)) | |
| splitlist= [] | |
| for s in info[dset_config].splits: | |
| if s != 'train': | |
| splitlist.append(s) | |
| dset_split = st.selectbox('Choose a dataset split for evaluation', options=splitlist) | |
| st.markdown("### Select up to 5 models to compare their performance:") | |
| filt = ModelFilter(trained_dataset=dset) | |
| all_models = [m.modelId for m in api.list_models(filter=filt, sort = "downloads", direction=-1, limit = 20) if 't5' not in m.tags] | |
| models = st.multiselect( | |
| 'Choose the models that have been trained/finetuned on this dataset', | |
| options=all_models) | |
| if len(models) > 5: | |
| st.exception("Please choose less than 5 models!") | |
| st.markdown("### What two metrics do you want to compare?") | |
| metrics = st.multiselect( | |
| 'Choose the metrics for the comparison', | |
| options=['f1', 'accuracy', 'precision', 'recall'], | |
| default=["f1", "accuracy"]) | |
| st.markdown("### Please wait for the dataset and models to load (this can take some time if they are big!") | |
| ### Loading data | |
| def loaddset(d, d_split): | |
| data = datasets.load_dataset(d, split=d_split) | |
| return(data) | |
| data = loaddset(dset,dset_split) | |
| ### Defining Evaluator | |
| eval = evaluator("text-classification") | |
| ### Loading models | |
| def load_models(mod_names): | |
| model_list=[] | |
| for i in range (len(mod_names)): | |
| try: | |
| globals()[f"tokenizer_{i}"] = AutoTokenizer.from_pretrained(mod_names[i]) | |
| globals()[f"model_{i}"] = AutoModelForSequenceClassification.from_pretrained(mod_names[i]) | |
| model_list.append(mod_names[i]) | |
| except: | |
| continue | |
| return(model_list) | |
| ### Defining pipelines | |
| def load_pipes(mod_list): | |
| pipe_list=[] | |
| for i in range (len(mod_list)): | |
| globals()[f"pipe_{i}"] = pipeline("text-classification", model = models[i], tokenizer = models[i], device=-1) | |
| return(pipe_list) | |
| model_list= load_models(models) | |
| pipes = load_pipes(model_list) | |
| ### Defining metrics | |
| for i in range (len(metrics)): | |
| globals()[f"metrics[i]"] = evaluate.load(metrics[i]) | |
| ## Label mapping | |
| st.markdown("### Help us pick the right labels for your models") | |
| st.text("The labels for your dataset are: "+ str(data.features['label'].names)) | |
| for i in range (len(model_list)): | |
| st.text("The labels for " + str(model_list[i]) + "are: "+ str(AutoConfig.from_pretrained(model_list[i]).id2label)) | |
| for j in range (len(data.features['label'].names)): | |
| globals()[f"model[i]_label[j]"] = st.selectbox("The label corresponding to **" + str(data.features['label'].names[i]) + "** is:", AutoConfig.from_pretrained(model_list[i]).id2label) | |
| _ = """ | |
| res_accuracy1 = eval.compute(model_or_pipeline=pipe1, data=data, metric=accuracy, | |
| label_mapping={"NEGATIVE": 0, "POSITIVE": 1},) | |
| res_f11 = eval.compute(model_or_pipeline=pipe1, data=data, metric=f1, | |
| label_mapping={"NEGATIVE": 0, "POSITIVE": 1},) | |
| print({**res_accuracy1, **res_f11}) | |
| pipe2 = pipeline("text-classification", model=model2, tokenizer= tokenizer2, device=0) | |
| res_accuracy2 = eval.compute(model_or_pipeline=pipe2, data=data, metric=accuracy, | |
| label_mapping={"LABEL_0": 0, "LABEL_1": 1},) | |
| res_f12 = eval.compute(model_or_pipeline=pipe2, data=data, metric=f1, | |
| label_mapping={"LABEL_0": 0, "LABEL_1": 1},) | |
| print({**res_accuracy2, **res_f12}) | |
| pipe3 = pipeline("text-classification", model=model3, tokenizer= tokenizer3, device=0) | |
| res_accuracy3 = eval.compute(model_or_pipeline=pipe3, data=data, metric=accuracy, | |
| label_mapping={"neg": 0, "pos": 1},) | |
| res_f13 = eval.compute(model_or_pipeline=pipe3, data=data, metric=f1, | |
| label_mapping={"neg": 0, "pos": 1},) | |
| print({**res_accuracy3, **res_f13}) | |
| pipe4 = pipeline("text-classification", model=model4, tokenizer= tokenizer4, device=0) | |
| res_accuracy4 = eval.compute(model_or_pipeline=pipe4, data=data, metric=accuracy, | |
| label_mapping={"LABEL_0": 0, "LABEL_1": 1},) | |
| res_f14 = eval.compute(model_or_pipeline=pipe4, data=data, metric=f1, | |
| label_mapping={"LABEL_0": 0, "LABEL_1": 1},) | |
| print({**res_accuracy4, **res_f14}) | |
| pipe5 = pipeline("text-classification", model=model5, tokenizer= tokenizer5, device=0) | |
| res_accuracy5 = eval.compute(model_or_pipeline=pipe5, data=data, metric=accuracy, | |
| label_mapping={"LABEL_0": 0, "LABEL_1": 1},) | |
| res_f15 = eval.compute(model_or_pipeline=pipe5, data=data, metric=f1, | |
| label_mapping={"LABEL_0": 0, "LABEL_1": 1},) | |
| print({**res_accuracy5, **res_f15}) | |
| plt.plot(res_accuracy1['accuracy'], res_f11['f1'], marker='o', markersize=6, color="red") | |
| plt.annotate('distilbert', xy=(res_accuracy1['accuracy']+0.001, res_f11['f1'])) | |
| plt.plot(res_accuracy2['accuracy'], res_f12['f1'], marker='o', markersize=6, color="blue") | |
| plt.annotate('distilbert-base-uncased-finetuned', xy=(res_accuracy2['accuracy']+0.001, res_f12['f1'])) | |
| plt.plot(res_accuracy3['accuracy'], res_f13['f1'], marker='o', markersize=6, color="green") | |
| plt.annotate('roberta-base', xy=(res_accuracy3['accuracy']-0.009, res_f13['f1'])) | |
| plt.plot(res_accuracy4['accuracy'], res_f14['f1'], marker='o', markersize=6, color="purple") | |
| plt.annotate('funnel-transformer-small', xy=(res_accuracy4['accuracy']-0.015, res_f14['f1'])) | |
| plt.plot(res_accuracy5['accuracy'], res_f15['f1'], marker='o', markersize=6, color="black") | |
| plt.annotate('SENATOR', xy=(res_accuracy5['accuracy']+0.001, res_f15['f1'])) | |
| plt.xlabel('Accuracy') | |
| plt.ylabel('F1 Score') | |
| #plt.xlim([0.9, 1.0]) | |
| #plt.ylim([0.9, 1.0]) | |
| plt.title('Comparing the Models') | |
| """ |