Repository: stanford-crfm/helm
Issue: IndexError: list index out of range
Root cause: Validation Gap

## Davinci (`OpenAI error: That model is currently overloaded with other requests. You can retry your request, or contact support@openai.com if the error persists. (Please include the request ID 1c1cbc4835bf74ad0726aa02b966c27b in your email.)`)

```
  Error when running quac:model=text,data_augmentation=canonical:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range

```

```
  Error when running ice:model=full_functionality_text,subset=HK:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

```
  Error when running ice:model=full_functionality_text,subset=IND:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

More informative error:

```
benchmark.executor.ExecutorError: Failed to make request to openai after retrying 1 times. Error: OpenAI error: That model is currently overloaded with other requests. You can retry your request, or contact support@openai.com if the error persists. (Please include the request ID 1c1cbc4835bf74ad0726aa02b966c27b in your email.)Request: Request(model='openai/davinci', prompt='
<\\/*\\*> line from left to right<\\/*\\*>\n\n\n  Jurisdiction of the Collector of Central Excise  vis-a-vis Section 11A of the Central Excises \n& Salt Act, 1944   <\\/*\\*> line from left to right<\\/*\\*> \n\n\nGODFREY W. PIMENTA, Officer-Central Excise, Pfizer Ltd., Bombay.\n\n \n\n  AFTER   the enactment of\nCentral Excises and Salt ( Amendment ) Act, 1985, the office of the Collector\nof Central Excise has assumed greater importance.\n\n\nOrdinarily, under the Act, the show cause notice has to be issued\nwithin six months from the relevant date.\n\n\nHowever, proviso to section 11A ( I) stipulates that a notice can be\nissued within five years from the relevant date if, the non-levy, short levy,\nor erroneous refund has occurred on account of any of the following\nconditions viz., - -  ( a)  fraud;  ( b)  collusion  (\nc)  wilful mis-statement or suppression of facts; and  ( d) \ncontraventions of any of the provisions of the Central Excises and Salt Act,\n1944, or rules made thereunder, with an intent to evade payment of duty. \n\n \n\nThe notice which is contemplated under proviso to section 11A ( 1)\nis mandatory in nature and condition precedent before any action is to be\ntaken under the said proviso, for recovery of duties short levied or short\npaid, erroneously refunded for breach of any of the conditions laid down\ntherein, which seeks to invoke the extended period of limitation specifically\ndevised by the Parliament. \n\n \n\nThis extraordinary power of invoking the extended period of\nlimitation of five years under proviso to section 11A(1) now vests in the\nCollector of Central Excise by virtue of the Central Excises and Salt (\nAmendment ) Act, 1985.\n\n\nThe object of bringing about this amendment is made clear in the\nStatement of Objects and Reasons appended to the Central Excises and Salt (\nAmendment ) Bill, 1985, which states that, show cause notice in regard to\nduty of excise short levied or short paid, etc. by reason of fraud,\ncollusion, or any wilful mis-statement or suppression of facts should be\nissued and such cases decided by the Collector of Central Excise instead of\nAssistant Collector of Central Excise. \n\n \n\nNow the question of law that arises and which is subject matter of\nconflicting decisions of various tribunals and Courts, is that whether the\nCollector includes Additional Collector as contemplated in proviso to section\n11A ( 1) to the Act.\n\n\nBut unfortunately, there is no clear cut decision on this vital\npoint, which has resulted in controversies between the assessees and the\nExcise Department. \n\n \n\nThe word  ` Collector\'  is not defined in the Act,\nbut rule 2 (  2  ) of the Central Excise Rules, 1944, defines\nthe word which means: ( A) in relation to excisable goods other than Salt - -\n<\\/*\\*> dotted line<\\/*\\*> ( B) in relation to Salt, the Salt Commissioner; and\nincludes Additional Collector and any officer specifically authorised under\nrule 4 or rule 5 to exercise throughout any State or any specified area\ntherein all or any of the powers of a Collector under these rules. \n\n \n\nThe definition of Collector as defined in rule 2 (  2\n ) of the Central Excise Rules, 1944, was a matter of controversy\nbetween various decisions of tribunals,   inter alia \n, " Khatri Biri Works, Beawar v. Collector of Central Excise ;\nJaipur "  [ 1983 E.L.T. 645 ( C.E.G.A.T. ) ;  "\nSunil Kumar Ghosh v. Collector of Central excise & Customs, West Bengal,\nCalcutta "  [ 1983 E.L.T. 1964 ( C.E.G.A.T. ) ; and \n" S. Kumar and Others v. Collector of Central Excise and others\n"  ( 1983 E.L.T. 1057 ( C.E.G.A.T. ) ] \n\n \n\nThis controversy was laid to rest by the decision of Calcutta\nHigh Court which examined the issue in detail in case of  "\nBansal Industrial Gases ( Bihar ) Ltd. v. Collector of Central Excise\n"  [ 1988 ( 37 ) E.L.T. 347 ( CAL. ) ] , wherein the Court has\nheld that the post of Additional Collector of Central Excise is the same and\nequivalent to the post of Collector. \n\n \n\nHowever, it is to be noted that the decision of Calcutta High\nCourt referred to above was in the context of sections 33, 35 and 35B of the\nAct, which is not the issue at present. \n\n \n\nThere is an equally applicable decision of the Special Bench in\nthe case of  " Mahavir Products v. Collector of Central Excise,\n"  [ 1989 ( 41 ) E.L.T. 491 ( Tribunal ) ] , wherein the\nappellant had raised the preliminary objection on the ground that after the\nAmendment Act of 1985, the Collector was not empowered under the Act to have\nthe proceedings transferred to himself from the Additional Collector.\n\n\nHowever, while negativating the above contention of the Appellant,\nthe Tribunal observed as follows:\n\n\n " The objection is not valid for two reasons.\n\n\n First,  what happened on 27. 12. l985 was that by amendment\nof sections 11A ( I) and ( 2), Collector became the competent adjudicating\nauthority in cases in which proviso to section 11 A ( I) ( for extending the\nlimit of 5 years ) had been invoked.\n\n\nSince in the present case, the said proviso had been invoked, there\nwas nothing wrong in transferring the proceedings to the Collector.\n\n\n Secondly,  both the Additional Collector and the Collector\nhad jurisdiction to adjudicate in the matter.\n\n\nBoth were proper officers.\n\n\nBoth were competent adjudicating authorities under section 33 and\nfurther under rule 2 (  2  ) ,  ` Collector\' \nby definition, included Additional Collector as well.\n\n\nThe appellants cannot complain of any prejudice or illegality so long\nas their case was adjudicated upon by another officer having jurisdiction\nafter due notice to them and after granting them a fresh personal hearing.\n\n\nThere is no merit in the  point that the\nproceedings before the Collector were without jurisdiction"  . \n\n \n\nHowever, it is respectfully submitted that, the observation of\nCEGAT in case of  " Mahavir Products "  supra is\nnot correct, because in the statement of Objects and Reasons of the Central\nExcises and Salt ( Amendment ) Bill, 1985, it was specifically and\ncategorically stated that, show cause notice in regard to duty of excise\nshort levied or short paid, etc. by reason of fraud, collusion, or any wilful\nmis-statement or suppression of facts  should be\nissued and such cases decided by the Collector of Central Excise instead of\nAssistant Collector. \n\n \n\nIn this connection, it is relevant to mention the decision of the\nSupreme Court in the case of  " Rib Tapes ( India) Pvt. Ltd\n" and Another " v. Union of India and Others, "\n [ 1986 ( 26 ) ELT 193 ( S.C. ) ] herein it was held:  " It is\nnot in dispute that in order to interpret a particular provision and to infer\nintention of the Legislature, the Objects and Reasons stated in the Bill,\nwhen it is presented to the Legislature could be used"  . \n\n \n\nIt, therefore, follows that the Collector of Central Excise, who\nhas been named the only person in the Central Excises and Salt ( Amendment )\nBill, 1985, is the sole authority in whom the power to issue show cause\nnotice as well as to adjudicate upon the same under proviso to section 11 A (\n1) is vested after the amendment. \n\n \n\nHowever, it is to be borne in mind that the Central Board of\nExcise and Customs, in exercise of the powers conferred by clause ( b) of\nsection 2 of the Central Excises and Salt Act, 1944, and rule 4 of the\nCentral Excise Rules, I944, has been empowered to appoint such officers as it\ndeems fit and invest in him such powers as may be specified in the\nnotification issued in this behalf. \n\n \n\nThis provision as contained in the Act and rules made thereunder\ncannot override the express provision of proviso to section 11 A ( 1).\n\n\nThe Additional Collector of Central Excise, who is an authority\nsubordinate to the Collector of Central Excise cannot usurp the jurisdiction\nof the Collector for the purpose of proviso to section 11 A ( 1).\n\n\nIf that was the intention in law, then Parliament would have so\nexpressed it.\n\n\nHowever, it is respectfully stated that as per proviso to section 11\nA ( 1), the power has been exclusively vested in the hands of the  "\nCollector"  and Parliament has not anywhere in', temperature=0, num_completions=1, top_k_per_token=1, max_tokens=0, stop_sequences=[], echo_prompt=True, top_p=1, presence_penalty=0, frequency_penalty=0, random=None)
```


## Instruct Curie (https://github.com/stanford-crfm/benchmarking/issues/565)

```
  Error when running msmarco:task=passage,model=full_functionality_text,track=trec,use_qrels_passages=True,use_topk_passages=True,valid_topk=30:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 139, in evaluate
    metric_per_instance_stats = metric_function(request_state_groups, train_trial_index)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 376, in information_retrieval
    run = self.get_run_ranking_dict_function(request_state_groups)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 345, in get_run_ranking_dict_binary_logprob_mode
    sorted_pids = self.get_sorted_oids(group)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 329, in get_sorted_oids
    token = request_state.result.completions[0].tokens[0]
IndexError: list index out of range
```

```
  Error when running msmarco:task=passage,model=full_functionality_text,track=trec,use_qrels_passages=False,use_topk_passages=True,valid_topk=30:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 116, in run_one
    metric_result: MetricResult = metric.evaluate(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 139, in evaluate
    metric_per_instance_stats = metric_function(request_state_groups, train_trial_index)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 376, in information_retrieval
    run = self.get_run_ranking_dict_function(request_state_groups)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 345, in get_run_ranking_dict_binary_logprob_mode
    sorted_pids = self.get_sorted_oids(group)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/multiple_request_metrics.py", line 329, in get_sorted_oids
    token = request_state.result.completions[0].tokens[0]
IndexError: list index out of range
```

## J1-Jumbo

```
  Error when running msmarco:task=passage,model=full_functionality_text,track=trec,use_qrels_passages=True,use_topk_passages=True,valid_topk=30:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

```
  Error when running msmarco:task=passage,model=full_functionality_text,track=trec,use_qrels_passages=False,use_topk_passages=True,valid_topk=30:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

```
  Error when running entity_matching:model=text,dataset=Beer,data_augmentation=canonical:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

```
  Error when running entity_data_imputation:model=text,dataset=Buy,data_augmentation=canonical:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

```
  Error when running twitter_aae:model=full_functionality_text,demographic=aa:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

```
  Error when running twitter_aae:model=full_functionality_text,demographic=white:
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 126, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 73, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 79, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 103, in run_one
    scenario_state = self.executor.execute(scenario_state)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 97, in execute
    request_states = list(
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
    yield fs.pop().result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 444, in result
    return self.__get_result()
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/u/nlp/anaconda/main/anaconda3/envs/crfm_benchmarking/lib/python3.8/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 92, in process
    tqdm.write(render_request_state(state))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/executor.py", line 75, in render_request_state
    pred_output = state.result.completions[0].text
IndexError: list index out of range
```

Debugged and created follow-up issues.