Spaces:
Runtime error
Runtime error
| # Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """TODO: Add a description here.""" | |
| import datasets | |
| import evaluate | |
| # TODO: Add BibTeX citation | |
| from ocr_evaluation.evaluate.metrics import evaluate_by_symbols, evaluate_by_words, evaluate_by_word_groups | |
| from ocr_evaluation.ocr.fiftyone import FiftyOneOcr | |
| _CITATION = """\ | |
| @InProceedings{huggingface:module, | |
| title = {Iliauni ICC OCR Evaluation}, | |
| authors={}, | |
| year={2022} | |
| } | |
| """ | |
| # TODO: Add description of the module here | |
| _DESCRIPTION = """\ | |
| Better OCR evaluation metric that enables to evaluate OCR results in various ways. It is robust in a way that | |
| it matches the words using their bounding boxes instead of using plain edit distance matching between two texts. | |
| Elaborate more on this later. | |
| """ | |
| # TODO: Add description of the arguments of the module here | |
| _KWARGS_DESCRIPTION = """ | |
| Calculates how good are predictions given some references, using certain scores | |
| Args: | |
| predictions: list of OCR detections in FiftyOne dataset format. | |
| references: list of OCR detections in FiftyOne dataset format. | |
| Returns: | |
| evaluation_results: list of dictionaries containing multiple metrics | |
| Examples: | |
| Examples should be written in doctest format, and should illustrate how | |
| to use the function. | |
| >>> dataset = load_dataset("anz2/iliauni_icc_georgian_ocr", use_auth_token="<auth token here>") | |
| >>> sample = dataset['test'][0] | |
| >>> ocr_evaluator = evaluate.load("anz2/iliauniiccocrevaluation") | |
| >>> results = ocr_evaluator._compute(predictions=[sample], references=[sample]) | |
| >>> print(results[0].keys()) | |
| dict_keys(['accuracy', 'precision', 'recall', 'f1', 'levenstein_distances_stats', 'levenstein_similarities_stats', 'iou_stats', 'edit_operations_stats']) | |
| """ | |
| class IliauniIccOCREvaluation(evaluate.Metric): | |
| """TODO: Short description of my evaluation module.""" | |
| def _info(self): | |
| # TODO: Specifies the evaluate.EvaluationModuleInfo object | |
| return evaluate.MetricInfo( | |
| # This is the description that will appear on the modules page. | |
| module_type="metric", | |
| description=_DESCRIPTION, | |
| citation=_CITATION, | |
| inputs_description=_KWARGS_DESCRIPTION, | |
| # This defines the format of each prediction and reference | |
| features=datasets.Features( | |
| { | |
| "predictions": datasets.Sequence( | |
| feature=datasets.Features( | |
| { | |
| "id": datasets.Value("string"), | |
| "filepath": datasets.Value("string"), | |
| "tags": datasets.Sequence(datasets.Value("string")), | |
| "metadata": datasets.Features( | |
| { | |
| "size_bytes": datasets.Value("int32"), | |
| "mime_type": datasets.Value("string"), | |
| "width": datasets.Value("int32"), | |
| "height": datasets.Value("int32"), | |
| "num_channels": datasets.Value("int32"), | |
| "author": datasets.Value("string"), | |
| "category": datasets.Value("string"), | |
| "document_name": datasets.Value("string"), | |
| "source": datasets.Value("string"), | |
| "year": datasets.Value("int32") | |
| } | |
| ), | |
| "_media_type": datasets.Value("string"), | |
| "_rand": datasets.Value("string"), | |
| "detections": datasets.Features( | |
| { | |
| "detections": datasets.Sequence( | |
| datasets.Features( | |
| { | |
| "id": datasets.Value("string"), | |
| "attributes": datasets.Sequence(datasets.Value("string")), | |
| "tags": datasets.Value("string"), | |
| "label": datasets.Value("string"), | |
| "bounding_box": datasets.Sequence(datasets.Value("float32")), | |
| "confidence": datasets.Value("float32"), | |
| "index": datasets.Value("int32"), | |
| "page": datasets.Value("int32"), | |
| "block": datasets.Value("int32"), | |
| "paragraph": datasets.Value("int32"), | |
| "word": datasets.Value("int32"), | |
| "text": datasets.Value("string"), | |
| } | |
| ) | |
| ) | |
| } | |
| ), | |
| "image": datasets.Image() | |
| } | |
| ) | |
| ), | |
| "references": datasets.Sequence( | |
| feature=datasets.Features( | |
| { | |
| "id": datasets.Value("string"), | |
| "filepath": datasets.Value("string"), | |
| "tags": datasets.Sequence(datasets.Value("string")), | |
| "metadata": datasets.Features( | |
| { | |
| "size_bytes": datasets.Value("int32"), | |
| "mime_type": datasets.Value("string"), | |
| "width": datasets.Value("int32"), | |
| "height": datasets.Value("int32"), | |
| "num_channels": datasets.Value("int32"), | |
| "author": datasets.Value("string"), | |
| "category": datasets.Value("string"), | |
| "document_name": datasets.Value("string"), | |
| "source": datasets.Value("string"), | |
| "year": datasets.Value("int32") | |
| } | |
| ), | |
| "_media_type": datasets.Value("string"), | |
| "_rand": datasets.Value("string"), | |
| "detections": datasets.Features( | |
| { | |
| "detections": datasets.Sequence( | |
| datasets.Features( | |
| { | |
| "id": datasets.Value("string"), | |
| "attributes": datasets.Sequence(datasets.Value("string")), | |
| "tags": datasets.Value("string"), | |
| "label": datasets.Value("string"), | |
| "bounding_box": datasets.Sequence(datasets.Value("float32")), | |
| "confidence": datasets.Value("float32"), | |
| "index": datasets.Value("int32"), | |
| "page": datasets.Value("int32"), | |
| "block": datasets.Value("int32"), | |
| "paragraph": datasets.Value("int32"), | |
| "word": datasets.Value("int32"), | |
| "text": datasets.Value("string"), | |
| } | |
| ) | |
| ) | |
| } | |
| ), | |
| "image": datasets.Image() | |
| } | |
| ) | |
| ) | |
| } | |
| ), | |
| # Homepage of the module for documentation | |
| homepage="", | |
| # Additional links to the codebase or references | |
| codebase_urls=["https://github.com/IliaUni-ICC/ocr_evaluation"], | |
| reference_urls=[] | |
| ) | |
| def _download_and_prepare(self, dl_manager): | |
| """Optional: download external resources useful to compute the scores""" | |
| # TODO: Download external resources if needed | |
| pass | |
| def _compute(self, *, predictions=None, references=None, **kwargs): | |
| """Returns the scores""" | |
| eval_method = "word" | |
| if kwargs.get("eval_method", "word") in ["symbol", "word", "word_group"]: | |
| eval_method = kwargs["eval_method"] | |
| assert len(predictions) == len(references) | |
| results = [] | |
| for prediction, reference in zip(predictions, references): | |
| prediction_df = FiftyOneOcr(data=prediction).get_detections(convert_bbox=True) | |
| reference_df = FiftyOneOcr(data=reference).get_detections(convert_bbox=True) | |
| if eval_method == "symbol": | |
| result = evaluate_by_symbols(prediction_df, reference_df, pref1="Pred_", pref2="Tar_") | |
| elif eval_method == "word": | |
| result = evaluate_by_words(prediction_df, reference_df, pref1="Pred_", pref2="Tar_") | |
| elif eval_method == "word_group": | |
| result = evaluate_by_word_groups(prediction_df, reference_df, pref1="Pred_", pref2="Tar_") | |
| else: | |
| result = {} | |
| results.append(result) | |
| return results | |