Repository: stanford-crfm/helm
Issue: `code:model=code,dataset=HumanEval` failed
Root cause: Validation Gap

  Error when running code:model=code,dataset=HumanEval:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 75, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 89, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 104, in run_one
    instances: List[Instance] = DataPreprocessor(run_spec.data_augmenter_spec).preprocess(scenario)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/data_preprocessor.py", line 30, in preprocess
    instances: Sequence[Instance] = scenario.get_instances()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/code_scenario.py", line 276, in get_instances
    instances = _read_and_preprocess_human_eval(target_path=target_path, **self.human_eval_hparams)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/code_scenario.py", line 83, in _read_and_preprocess_human_eval
    problems = _read_human_eval(target_path)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/code_scenario.py", line 107, in _read_human_eval
    return {task["task_id"]: task for task in _stream_jsonl(evalset_file)}
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/code_scenario.py", line 107, in <dictcomp>
    return {task["task_id"]: task for task in _stream_jsonl(evalset_file)}
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/code_scenario.py", line 113, in _stream_jsonl
    for line in f:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte

Verified it works with the fix:

      MetricName(name='code_eval_acc', k=None, split='test', sub_split=None, perturbation=None)[min=0.317, mean=0.317, max=0.317, sum=0.317 (1)]
      MetricName(name='code_eval_acc', k=1, split='test', sub_split=None, perturbation=None)[min=0.317, mean=0.317, max=0.317, sum=0.317 (1)]
      MetricName(name='pass', k=None, split='test', sub_split=None, perturbation=None)[min=0.317, mean=0.317, max=0.317, sum=0.317 (1)]
      MetricName(name='pass', k=1, split='test', sub_split=None, perturbation=None)[min=0.317, mean=0.317, max=0.317, sum=0.317 (1)]
      MetricName(name='logprob', k=None, split='test', sub_split=None, perturbation=None)[min=-16.092, mean=-16.092, max=-16.092, sum=-16.092 (1)]
      MetricName(name='num_perplexity_tokens', k=None, split='test', sub_split=None, perturbation=None)[min=88.683, mean=88.683, max=88.683, sum=88.683 (1)]
      MetricName(name='num_bytes', k=None, split='test', sub_split=None, perturbation=None)[min=251.890, mean=251.890, max=251.890, sum=251.890 (1)]
      MetricName(name='num_output_tokens', k=None, split='test', sub_split=None, perturbation=None)[min=88.683, mean=88.683, max=88.683, sum=88.683 (1)]
      MetricName(name='inference_runtime', k=None, split='test', sub_split=None, perturbation=None)[min=0.917, mean=0.917, max=0.917, sum=0.917 (1)]
      MetricName(name='inference_idealized_runtime', k=None, split='test', sub_split=None, perturbation=None)[(0)]
      MetricName(name='inference_runtime_discrepancy', k=None, split='test', sub_split=None, perturbation=None)[(0)]
      MetricName(name='training_co2_cost', k=None, split='test', sub_split=None, perturbation=None)[(0)]
      MetricName(name='finish_reason_length', k=None, split='test', sub_split=None, perturbation=None)[min=0.049, mean=0.049, max=0.049, sum=0.049 (1)]
      MetricName(name='finish_reason_stop', k=None, split='test', sub_split=None, perturbation=None)[min=0.951, mean=0.951, max=0.951, sum=0.951 (1)]
      MetricName(name='finish_reason_endoftext', k=None, split='test', sub_split=None, perturbation=None)[min=0.000, mean=0.000, max=0.000, sum=0.000 (1)]
      MetricName(name='finish_reason_unknown', k=None, split='test', sub_split=None, perturbation=None)[min=0.000, mean=0.000, max=0.000, sum=0.000 (1)]
      MetricName(name='num_in_context_examples', k=None, split='test', sub_split=None, perturbation=None)[min=0.000, mean=0.000, max=0.000, sum=0.000 (1)]
      MetricName(name='num_instances', k=None, split='test', sub_split=None, perturbation=None)[min=164.000, mean=164.000, max=164.000, sum=164.000 (1)]
      MetricName(name='num_instances', k=1, split='test', sub_split=None, perturbation=None)[min=164.000, mean=164.000, max=164.000, sum=164.000 (1)]
      MetricName(name='perplexity', k=None, split='test', sub_split=None, perturbation=None)[min=1.199, mean=1.199, max=1.199, sum=1.199 (1)]
      MetricName(name='bits_per_byte', k=None, split='test', sub_split=None, perturbation=None)[min=0.092, mean=0.092, max=0.092, sum=0.092 (1)]
      MetricName(name='logprob_per_byte', k=None, split='test', sub_split=None, perturbation=None)[min=-0.064, mean=-0.064, max=-0.064, sum=-0.064 (1)]
      MetricName(name='estimated_num_tokens_cost', k=None, split=None, sub_split=None, perturbation=None)[min=654.000, mean=770.348, max=1228.000, sum=126337.000 (164)]
      MetricName(name='num_prompt_tokens', k=None, split=None, sub_split=None, perturbation=None)[min=54.000, mean=170.348, max=628.000, sum=27937.000 (164)]
      MetricName(name='max_num_output_tokens', k=None, split=None, sub_split=None, perturbation=None)[min=600.000, mean=600.000, max=600.000, sum=98400.000 (164)]
      MetricName(name='num_requests', k=None, split=None, sub_split=None, perturbation=None)[min=164.000, mean=164.000, max=164.000, sum=164.000 (1)]
    } [0.001s]
    Writing 1239 characters to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/run_spec.json
    Writing 824 lines to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/scenario.txt
    Writing 161898 characters to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/scenario.json
    Writing 22877 lines to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/scenario_state.txt
    Writing 3308706 characters to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/scenario_state.json
    Writing 26 lines to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/metrics.txt
    Writing 16174 characters to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/metrics.json
    Writing 1219124 characters to benchmark_output/runs/07-26-2022/code:dataset=HumanEval,model=openai_code-cushman-001/per_instance_metrics.json


Thanks @lxuechen!

Glad it worked!