Spaces:
Running
Running
File size: 4,866 Bytes
44921ac |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 |
import pandas as pd
import streamlit as st
from annotated_text import annotated_text
from annotated_text.util import get_annotated_html
from streamlit_annotation_tools import text_labeler
from evaluation_metrics import EVALUATION_METRICS, get_evaluation_metric
from predefined_example import EXAMPLES
from span_dataclass_converters import (
get_highlight_spans_from_ner_spans,
get_ner_spans_from_annotations,
)
@st.cache_resource
def get_examples_attributes(selected_example):
"Return example attributes so that they are not refreshed on every interaction"
return (
selected_example.text,
selected_example.gt_labels,
selected_example.gt_spans,
selected_example.predictions,
)
if __name__ == "__main__":
st.set_page_config(layout="wide")
st.title("NER Evaluation Metrics Comparison")
st.write(
"Evaluation for the NER task requires a ground truth and a prediction that will be evaluated. The ground truth is shown below, add predictions in the next section to compare the evaluation metrics."
)
# with st.container():
st.subheader("Ground Truth") # , divider='rainbow')
selected_example = st.selectbox(
"Select an example text from the drop down below",
[example for example in EXAMPLES],
format_func=lambda ex: ex.text,
)
text, gt_labels, gt_spans, predictions = get_examples_attributes(selected_example)
annotated_text(
get_highlight_spans_from_ner_spans(
get_ner_spans_from_annotations(gt_labels), text
)
)
annotated_predictions = [
get_annotated_html(get_highlight_spans_from_ner_spans(ner_span, text))
for ner_span in predictions
]
predictions_df = pd.DataFrame(
{
# "ID": [f"Prediction_{index}" for index in range(len(predictions))],
"Prediction": annotated_predictions,
"ner_spans": predictions,
},
index=[f"Prediction_{index}" for index in range(len(predictions))],
)
st.subheader("Predictions") # , divider='rainbow')
with st.expander("Click to Add Predictions"):
st.subheader("Adding predictions")
st.markdown(
"""
Add predictions to the list of predictions on which the evaluation metric will be caculated.
- Select the entity type/label name and then highlight the span in the text below.
- To remove a span, double click on the higlighted text.
- Once you have your desired prediction, click on the 'Add' button.(The prediction created is shown in a json below)
"""
)
st.write(
"Note: Only the spans of the selected label name is shown at a given instance.",
)
labels = text_labeler(text, gt_labels)
st.json(labels, expanded=False)
# if st.button("Add Prediction"):
# labels = text_labeler(text)
if st.button("Add!"):
spans = get_ner_spans_from_annotations(labels)
spans = sorted(spans, key=lambda span: span["start"])
predictions.append(spans)
annotated_predictions.append(
get_annotated_html(get_highlight_spans_from_ner_spans(spans, text))
)
predictions_df = pd.DataFrame(
{
# "ID": [f"Prediction_{index}" for index in range(len(predictions))],
"Prediction": annotated_predictions,
"ner_spans": predictions,
},
index=[f"Prediction_{index}" for index in range(len(predictions))],
)
print("added")
highlighted_predictions_df = predictions_df[["Prediction"]]
st.write(highlighted_predictions_df.to_html(escape=False), unsafe_allow_html=True)
st.divider()
### EVALUATION METRICS COMPARISION ###
st.subheader("Evaluation Metrics Comparision") # , divider='rainbow')
st.markdown("""
The different evaluation metrics we have for the NER task are
- Span Based Evaluation with Partial Overlap
- Token Based Evaluation with Micro Avg
- Token Based Evaluation with Macro Avg
""")
with st.expander("View Predictions Details"):
st.write(predictions_df.to_html(escape=False), unsafe_allow_html=True)
if st.button("Get Metrics!"):
for evaluation_metric_type in EVALUATION_METRICS:
predictions_df[evaluation_metric_type] = predictions_df.ner_spans.apply(
lambda ner_spans: get_evaluation_metric(
metric_type=evaluation_metric_type,
gt_ner_span=gt_spans,
pred_ner_span=ner_spans,
text=text,
)
)
metrics_df = predictions_df.drop(["ner_spans"], axis=1)
st.write(metrics_df.to_html(escape=False), unsafe_allow_html=True)
print("compared")
|