jordyvl commited on
Commit
10e1e9c
1 Parent(s): 8ec938f

small update to support unrealistic perfect classification

Browse files
Files changed (1) hide show
  1. ece.py +9 -15
ece.py CHANGED
@@ -104,7 +104,7 @@ def create_bins(n_bins=10, scheme="equal-range", bin_range=None, P=None):
104
  def discretize_into_bins(P, bins):
105
 
106
  contains_rightmost = bool(bins[-1] > 1) # outlier bins
107
- contains_leftmost = bool(bins[0] == 0) # beyond [before] bin_range[0]
108
  # bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
109
 
110
  oneDbins = np.digitize(
@@ -112,7 +112,6 @@ def discretize_into_bins(P, bins):
112
  ) # since bins contains extra righmost (& leftmost bins)
113
  if contains_leftmost:
114
  oneDbins -= 1
115
- # oneDbins = np.digitize(P, bins) - 1 # since bins contains extra righmost (& leftmost bins)
116
 
117
  # Fix to scipy.binned_dd_statistic:
118
  # Tie-breaking to the left for rightmost bin
@@ -245,8 +244,8 @@ class ECE(evaluate.EvaluationModule):
245
  # Homepage of the module for documentation
246
  homepage="https://huggingface.co/spaces/jordyvl/ece",
247
  # Additional links to the codebase or references
248
- codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
249
- reference_urls=["http://path.to.reference.url/new_module"],
250
  )
251
 
252
  def init_kwargs(
@@ -341,32 +340,27 @@ def test_equalmass_binning():
341
  scheme="equal-mass",
342
  bin_range=None,
343
  proxy="upper-edge",
344
- # proxy="center",
345
  p=1,
346
  detail=True,
347
  )
348
  bins = create_bins(
349
  n_bins=kwargs["n_bins"], scheme=kwargs["scheme"], bin_range=kwargs["bin_range"], P=probs
350
  )
351
-
352
  test_ECE(**kwargs)
353
 
354
- """
 
 
355
  res = ECE()._compute(
356
- references=[0, 1, 2],
357
- predictions=[[0.63, 0.2, 0.2], [0, 0.95, 0.05], [0.72, 0.1, 0.2]],
358
  detail=True,
359
  )
360
  print(f"ECE: {res['ECE']}\n {res}")
361
- """
362
- # need to provide lens
363
-
364
- import pdb
365
-
366
- pdb.set_trace() # breakpoint 94583f9f //
367
 
368
 
369
  if __name__ == "__main__":
 
370
  test_equalmass_binning()
371
  test_deterministic()
372
  test_ECE()
 
104
  def discretize_into_bins(P, bins):
105
 
106
  contains_rightmost = bool(bins[-1] > 1) # outlier bins
107
+ contains_leftmost = bool(bins[0] <= 0) # beyond [before] bin_range[0]
108
  # bins_with_left_edge = np.insert(bins, 0, 0, axis=0)
109
 
110
  oneDbins = np.digitize(
 
112
  ) # since bins contains extra righmost (& leftmost bins)
113
  if contains_leftmost:
114
  oneDbins -= 1
 
115
 
116
  # Fix to scipy.binned_dd_statistic:
117
  # Tie-breaking to the left for rightmost bin
 
244
  # Homepage of the module for documentation
245
  homepage="https://huggingface.co/spaces/jordyvl/ece",
246
  # Additional links to the codebase or references
247
+ codebase_urls=[""],
248
+ reference_urls=[""],
249
  )
250
 
251
  def init_kwargs(
 
340
  scheme="equal-mass",
341
  bin_range=None,
342
  proxy="upper-edge",
 
343
  p=1,
344
  detail=True,
345
  )
346
  bins = create_bins(
347
  n_bins=kwargs["n_bins"], scheme=kwargs["scheme"], bin_range=kwargs["bin_range"], P=probs
348
  )
 
349
  test_ECE(**kwargs)
350
 
351
+
352
+ def test_perfect_predictions(K=3):
353
+ references = [0, 1, 2]
354
  res = ECE()._compute(
355
+ references=references,
356
+ predictions=np.eye(K)[references],
357
  detail=True,
358
  )
359
  print(f"ECE: {res['ECE']}\n {res}")
 
 
 
 
 
 
360
 
361
 
362
  if __name__ == "__main__":
363
+ test_perfect_predictions()
364
  test_equalmass_binning()
365
  test_deterministic()
366
  test_ECE()