init
Browse files- charmatch.py +20 -12
- requirements.txt +2 -1
charmatch.py
CHANGED
@@ -15,7 +15,7 @@
|
|
15 |
|
16 |
import evaluate
|
17 |
import datasets
|
18 |
-
|
19 |
|
20 |
# TODO: Add BibTeX citation
|
21 |
_CITATION = """\
|
@@ -66,13 +66,14 @@ class charmatch(evaluate.Metric):
|
|
66 |
return evaluate.MetricInfo(
|
67 |
# This is the description that will appear on the modules page.
|
68 |
module_type="metric",
|
69 |
-
description=
|
70 |
-
citation=
|
71 |
-
inputs_description=
|
72 |
# This defines the format of each prediction and reference
|
73 |
features=datasets.Features({
|
74 |
-
'
|
75 |
-
'
|
|
|
76 |
}),
|
77 |
# Homepage of the module for documentation
|
78 |
homepage="http://module.homepage",
|
@@ -86,10 +87,17 @@ class charmatch(evaluate.Metric):
|
|
86 |
# TODO: Download external resources if needed
|
87 |
pass
|
88 |
|
89 |
-
def _compute(self,
|
90 |
-
|
91 |
-
|
92 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
93 |
return {
|
94 |
-
"
|
95 |
-
}
|
|
|
15 |
|
16 |
import evaluate
|
17 |
import datasets
|
18 |
+
from Levenshtein import distance as lev
|
19 |
|
20 |
# TODO: Add BibTeX citation
|
21 |
_CITATION = """\
|
|
|
66 |
return evaluate.MetricInfo(
|
67 |
# This is the description that will appear on the modules page.
|
68 |
module_type="metric",
|
69 |
+
description="Charmatch",
|
70 |
+
citation="",
|
71 |
+
inputs_description="input expected output",
|
72 |
# This defines the format of each prediction and reference
|
73 |
features=datasets.Features({
|
74 |
+
'input': datasets.Value('string'),
|
75 |
+
'expected': datasets.Value('string'),
|
76 |
+
'output': datasets.Value('string')
|
77 |
}),
|
78 |
# Homepage of the module for documentation
|
79 |
homepage="http://module.homepage",
|
|
|
87 |
# TODO: Download external resources if needed
|
88 |
pass
|
89 |
|
90 |
+
def _compute(self, input, expected, output):
|
91 |
+
expected_corrections = lev(input, expected)
|
92 |
+
distance_to_input = lev(input, output)
|
93 |
+
distance_to_expected = lev(output, expected)
|
94 |
+
|
95 |
+
true_positives = min(expected_corrections, max(0, (expected_corrections + distance_to_input - distance_to_expected))) / 2
|
96 |
+
|
97 |
+
precision = true_positives / distance_to_input
|
98 |
+
recall = true_positives / expected_corrections
|
99 |
+
f_05 = (1 + 0.5**2) * (precision * recall) / (0.5**2 * precision + recall)
|
100 |
+
|
101 |
return {
|
102 |
+
"fscore": f_05
|
103 |
+
}
|
requirements.txt
CHANGED
@@ -1 +1,2 @@
|
|
1 |
-
git+https://github.com/huggingface/evaluate@main
|
|
|
|
1 |
+
git+https://github.com/huggingface/evaluate@main
|
2 |
+
levenshtein
|