mgfrantz commited on
Commit
de4a630
1 Parent(s): 9ef4782

initial commit

Browse files
Files changed (1) hide show
  1. roc_auc_macro.py +29 -23
roc_auc_macro.py CHANGED
@@ -15,47 +15,55 @@
15
 
16
  import evaluate
17
  import datasets
 
18
 
19
 
20
  # TODO: Add BibTeX citation
21
  _CITATION = """\
22
  @InProceedings{huggingface:module,
23
- title = {A great new module},
24
- authors={huggingface, Inc.},
25
- year={2020}
26
  }
27
  """
28
 
29
  # TODO: Add description of the module here
30
  _DESCRIPTION = """\
31
- This new module is designed to solve this great ML task and is crafted with a lot of care.
 
 
32
  """
33
 
34
 
35
  # TODO: Add description of the arguments of the module here
36
  _KWARGS_DESCRIPTION = """
37
- Calculates how good are predictions given some references, using certain scores
 
 
 
38
  Args:
39
- predictions: list of predictions to score. Each predictions
40
- should be a string with tokens separated by spaces.
 
 
41
  references: list of reference for each prediction. Each
42
- reference should be a string with tokens separated by spaces.
 
43
  Returns:
44
- accuracy: description of the first score,
45
- another_score: description of the second score,
46
  Examples:
47
  Examples should be written in doctest format, and should illustrate how
48
  to use the function.
49
 
50
- >>> my_new_module = evaluate.load("my_new_module")
51
- >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
 
 
 
52
  >>> print(results)
53
- {'accuracy': 1.0}
54
  """
55
 
56
- # TODO: Define external resources urls if needed
57
- BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
58
-
59
 
60
  @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
61
  class roc_auc_macro(evaluate.Metric):
@@ -71,14 +79,14 @@ class roc_auc_macro(evaluate.Metric):
71
  inputs_description=_KWARGS_DESCRIPTION,
72
  # This defines the format of each prediction and reference
73
  features=datasets.Features({
74
- 'predictions': datasets.Value('int64'),
75
  'references': datasets.Value('int64'),
76
  }),
77
  # Homepage of the module for documentation
78
- homepage="http://module.homepage",
79
  # Additional links to the codebase or references
80
- codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
81
- reference_urls=["http://path.to.reference.url/new_module"]
82
  )
83
 
84
  def _download_and_prepare(self, dl_manager):
@@ -88,8 +96,6 @@ class roc_auc_macro(evaluate.Metric):
88
 
89
  def _compute(self, predictions, references):
90
  """Returns the scores"""
91
- # TODO: Compute the different scores of the module
92
- accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
93
  return {
94
- "accuracy": accuracy,
95
  }
 
15
 
16
  import evaluate
17
  import datasets
18
+ from sklearn.metrics import roc_auc_score
19
 
20
 
21
  # TODO: Add BibTeX citation
22
  _CITATION = """\
23
  @InProceedings{huggingface:module,
24
+ title = {Roc AUC Macro},
25
+ authors={Mike Frantz},
26
+ year={2022}
27
  }
28
  """
29
 
30
  # TODO: Add description of the module here
31
  _DESCRIPTION = """\
32
+ This module makes scikit-learn's roc_auc_score with average='macro'
33
+ available as a metric in the hub for use in multi-class or multi-label
34
+ classification.
35
  """
36
 
37
 
38
  # TODO: Add description of the arguments of the module here
39
  _KWARGS_DESCRIPTION = """
40
+ Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores.
41
+ Calculates metrics for each label, and find their unweighted mean. This does not take label imbalance into account.
42
+ (FROM https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html)
43
+
44
  Args:
45
+ scores: An array of shape (n_samples, n_classes) of probability estimates.
46
+ In the multi-class case, the probabilities must sum to 1 across classes.
47
+ In the multi-label case, the probabilities, each output corresponds
48
+ to a binary decision for a particular label.
49
  references: list of reference for each prediction. Each
50
+ reference should be an array of shape (n_sampls, n_classes) with values
51
+ of 0 or 1.
52
  Returns:
53
+ roc_auc_macro
 
54
  Examples:
55
  Examples should be written in doctest format, and should illustrate how
56
  to use the function.
57
 
58
+ >>> roc_auc_macro = evaluate.load("mgfrantz/roc_auc_macro")
59
+ >>> results = my_new_module.compute(
60
+ references=[[1, 0, 1], [0, 1, 0]],
61
+ scores=[[.8, .2, .9], [.1, .8, .6]]
62
+ )
63
  >>> print(results)
64
+ {'roc_auc_macro': 1.0}
65
  """
66
 
 
 
 
67
 
68
  @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
69
  class roc_auc_macro(evaluate.Metric):
 
79
  inputs_description=_KWARGS_DESCRIPTION,
80
  # This defines the format of each prediction and reference
81
  features=datasets.Features({
82
+ 'scores': datasets.Value('float64'),
83
  'references': datasets.Value('int64'),
84
  }),
85
  # Homepage of the module for documentation
86
+ homepage="N/A",
87
  # Additional links to the codebase or references
88
+ codebase_urls=["N/A"],
89
+ reference_urls=["N/A"]
90
  )
91
 
92
  def _download_and_prepare(self, dl_manager):
 
96
 
97
  def _compute(self, predictions, references):
98
  """Returns the scores"""
 
 
99
  return {
100
+ "roc_auc_macro": roc_auc_score(references, scores, average='macro'),
101
  }