Repository: stanford-crfm/helm
Issue: Error when calculating standard deviation
Root cause: Validation Gap

A handful of AI21 runs failed:

j1-jumbo:

```
  Error when running ice:model=full_functionality_text,subset=HK:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/metric.py", line 74, in evaluate
    return self.evaluate_language_modeling(scenario_state, metric_service, eval_cache_path)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/metric.py", line 221, in evaluate_language_modeling
    merge_stat(trial_stats, stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 100, in merge_stat
    stats[stat.name].merge(stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 58, in merge
    self._add_to_values(x)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 33, in _add_to_values
    self.update_mean_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 86, in update_mean_stddev
    self.stddev = self.compute_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 81, in compute_stddev
    return math.sqrt(self.variance) if self.variance is not None else None
ValueError: math domain error
```

j1-large:

```
  Error when running copyright:model=text,datatag=n_books_1000-extractions_per_book_1-prefix_length_125:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/metric.py", line 112, in evaluate
    merge_stat(trial_stats, stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 100, in merge_stat
    stats[stat.name].merge(stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 58, in merge
    self._add_to_values(x)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 33, in _add_to_values
    self.update_mean_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 86, in update_mean_stddev
    self.stddev = self.compute_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 81, in compute_stddev
    return math.sqrt(self.variance) if self.variance is not None else None
ValueError: math domain error
```


```
  Error when running copyright:model=text,datatag=popular_books-prefix_length_125.json:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/metric.py", line 112, in evaluate
    merge_stat(trial_stats, stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 100, in merge_stat
    stats[stat.name].merge(stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 58, in merge
    self._add_to_values(x)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 33, in _add_to_values
    self.update_mean_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 86, in update_mean_stddev
    self.stddev = self.compute_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 81, in compute_stddev
    return math.sqrt(self.variance) if self.variance is not None else None
ValueError: math domain error
```

```
  Error when running entity_matching:model=text,dataset=Beer,data_augmentation=canonical:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/metric.py", line 112, in evaluate
    merge_stat(trial_stats, stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 100, in merge_stat
    stats[stat.name].merge(stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 58, in merge
    self._add_to_values(x)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 33, in _add_to_values
    self.update_mean_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 86, in update_mean_stddev
    self.stddev = self.compute_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 81, in compute_stddev
    return math.sqrt(self.variance) if self.variance is not None else None
ValueError: math domain error
```

```
  Error when running entity_data_imputation:model=text,dataset=Buy,data_augmentation=canonical:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/metric.py", line 125, in evaluate
    per_instance_stats[key].merge(stat)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 58, in merge
    self._add_to_values(x)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 33, in _add_to_values
    self.update_mean_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 86, in update_mean_stddev
    self.stddev = self.compute_stddev()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/statistic.py", line 81, in compute_stddev
    return math.sqrt(self.variance) if self.variance is not None else None
ValueError: math domain error
```

I am guessing variance is negative for some reason?

Yep, seems wild - taking a look...

Alright, seems to be a numerical stability error - we are getting a very tiny negative number because of the subtraction in variance calculation.