Repository: stanford-crfm/helm
Issue: 1000 eval instances run - AI21 errors
Root cause: External Dependency Breakage

## Large
wikifact:model=ai21/j1-large,k=5,subject=P2175
gsm:model=ai21/j1-large
babi_qa:model=ai21/j1-large,task=17
imdb:model=ai21/j1-large,data_augmentation=dialect_deterministic
civil_comments:model=ai21/j1-large,data_path=/u/scr/nlp/crfm/benchmarking/civil_comments/,subject=homosexual_gay_or_lesbian
civil_comments:model=ai21/j1-large,data_path=/u/scr/nlp/crfm/benchmarking/civil_comments/,subject=other_disability
civil_comments:model=ai21/j1-large,data_path=/u/scr/nlp/crfm/benchmarking/civil_comments/,subject=other_religion
civil_comments:model=ai21/j1-large,data_path=/u/scr/nlp/crfm/benchmarking/civil_comments/,subject=white
boolq:model=ai21/j1-large,data_augmentation=all
news_qa:model=ai21/j1-large,data_augmentation=all
imdb:model=ai21/j1-large,only_contrast=True,data_augmentation=contrast_sets

## Grande
natural_qa:model=ai21/j1-grande,mode=closedbook,data_augmentation=all (Disconnected error)
news_qa:model=ai21/j1-grande,data_augmentation=all
imdb:model=ai21/j1-grande,data_augmentation=gender_pronouns_deterministic

## Jumbo
quac:model=ai21/j1-jumbo
civil_comments:model=ai21/j1-jumbo,data_path=/u/scr/nlp/crfm/benchmarking/civil_comments/,subject=homosexual_gay_or_lesbianjumbo,data_path=/u/scr/nlp/crfm/benchmarking/civil_comments/,subject=homosexual_gay_or_lesbian
boolq:model=ai21/j1-jumbo,data_augmentation=all
news_qa:model=ai21/j1-jumbo,data_augmentation=all


```
Traceback (most recent call last):
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/presentation/present.py", line 110, in run
    new_run_specs = run_benchmarking(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/run.py", line 54, in run_benchmarking
    runner.run_all()
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 75, in run_all
    self.run_one(run_spec)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/runner.py", line 96, in run_one
    scenario_state: ScenarioState = adapter.adapt(instances)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/common/hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/adapter.py", line 331, in adapt
    prompt: str = self.construct_prompt(train_instances, eval_instance)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/benchmark/adapter.py", line 484, in construct_prompt
    if self.tokenizer.fits_within_context_window(
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/tokenizer/ai21_tokenizer.py", line 59, in fits_within_context_window
    return self.tokenize_and_count(text) + expected_completion_token_length <= self.max_sequence_length
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/tokenizer/ai21_tokenizer.py", line 56, in tokenize_and_count
    return len(self.tokenize(text))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/tokenizer/ai21_tokenizer.py", line 51, in tokenize
    response: TokenizationRequestResult = self._make_tokenization_request(text)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/tokenizer/ai21_tokenizer.py", line 90, in _make_tokenization_request
    return self.service.tokenize(TokenizationRequest(text=text, model=self.model))
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/tokenizer/tokenizer_service.py", line 20, in tokenize
    return self._service.tokenize(self._auth, request)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/remote_service.py", line 56, in tokenize
    RemoteService._check_response(response)
  File "/juice/scr/nlp/crfm/benchmarking/benchmarking/src/proxy/remote_service.py", line 29, in _check_response
    raise RemoteServiceError(response["error"])
proxy.remote_service.RemoteServiceError: EXCEPTION: Expecting value: line 1 column 1 (char 0)
  File "/home/benchmarking/src/proxy/server.py", line 52, in safe_call
    result = func(params)

  File "/home/benchmarking/src/proxy/server.py", line 162, in perform
    return dataclasses.asdict(service.tokenize(auth, request))

  File "/home/benchmarking/src/proxy/server_service.py", line 95, in tokenize
    return self.client.tokenize(request)

  File "/home/benchmarking/src/proxy/auto_client.py", line 83, in tokenize
    return client.tokenize(request)

  File "/home/benchmarking/src/proxy/ai21_client.py", line 128, in tokenize
    response, cached = self.cache.get(raw_request, do_it)

  File "/home/benchmarking/src/common/cache.py", line 54, in get
    cache[key] = response = compute()

  File "/home/benchmarking/src/proxy/ai21_client.py", line 117, in do_it
    response = requests.post(

  File "/home/benchmarking/venv/lib/python3.8/site-packages/requests/models.py", line 910, in json
    return complexjson.loads(self.text, **kwargs)

  File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
    return _default_decoder.decode(s)

  File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())

  File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None

```