Repository: EleutherAI/lm-evaluation-harness (language: python)

Issue:
`o1-preview` support

It seems that current version of `lm_eval` does not supported `o1` series models.

When I execute
```sh
lm_eval\
    --model openai-chat-completions\
    --model_args model="o1-preview"\
    --apply_chat_template\
    --tasks ${task}\
    --verbosity DEBUG\
    --log_samples\
    --output_path ${log_path}

```

I got:

```sh
2024-10-30:10:47:19,626 WARNING  [api_models.py:349] API request failed with error message: {
  "error": {
    "message": "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.",
    "type": "invalid_request_error",
    "param": "max_tokens",
    "code": "unsupported_parameter"
  }
}. Retrying...
Traceback (most recent call last):
  File "/home/zi/anaconda3/envs/robench/bin/lm_eval", line 8, in <module>
    sys.exit(cli_evaluate())
  File "/home/zi/harness-4-robench/lm_eval/__main__.py", line 382, in cli_evaluate
    results = evaluator.simple_evaluate(
  File "/home/zi/harness-4-robench/lm_eval/utils.py", line 397, in _wrapper
    return fn(*args, **kwargs)
  File "/home/zi/harness-4-robench/lm_eval/evaluator.py", line 301, in simple_evaluate
    results = evaluate(
  File "/home/zi/harness-4-robench/lm_eval/utils.py", line 397, in _wrapper
    return fn(*args, **kwargs)
  File "/home/zi/harness-4-robench/lm_eval/evaluator.py", line 500, in evaluate
    resps = getattr(lm, reqtype)(cloned_reqs)
  File "/home/zi/harness-4-robench/lm_eval/models/api_models.py", line 571, in generate_until
    outputs = retry(
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/tenacity/__init__.py", line 336, in wrapped_f
    return copy(f, *args, **kw)
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/tenacity/__init__.py", line 475, in __call__
    do = self.iter(retry_state=retry_state)
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/tenacity/__init__.py", line 376, in iter
    result = action(retry_state)
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/tenacity/__init__.py", line 418, in exc_check
    raise retry_exc.reraise()
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/tenacity/__init__.py", line 185, in reraise
    raise self.last_attempt.result()
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/concurrent/futures/_base.py", line 451, in result
    return self.__get_result()
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/tenacity/__init__.py", line 478, in __call__
    result = fn(*args, **kwargs)
  File "/home/zi/harness-4-robench/lm_eval/models/api_models.py", line 352, in model_call
    response.raise_for_status()
  File "/home/zi/anaconda3/envs/robench/lib/python3.10/site-packages/requests/models.py", line 1024, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: https://api.openai.com/v1/chat/completions
```

The error is `lm_eval` will send a default variable of `max_tokens` via openai's API while o1 doesn't support this parameter. 

I doesn't define `max_tokens` in `model_args` explictly, so how can I disable this option?








Discussion / hints:

Fix patch:
diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py
index 331a3f6af5..2f10316f75 100644
--- a/lm_eval/models/openai_completions.py
+++ b/lm_eval/models/openai_completions.py
@@ -126,6 +126,9 @@ def _create_payload(
         seed=1234,
         **kwargs,
     ) -> dict:
+        assert (
+            type(messages) is not str
+        ), "chat-completions require the --apply_chat_template flag."
         gen_kwargs.pop("do_sample", False)
         if "max_tokens" in gen_kwargs:
             max_tokens = gen_kwargs.pop("max_tokens")
@@ -217,6 +220,10 @@ def __init__(
         tokenized_requests=False,
         **kwargs,
     ):
+        if "o1" in kwargs.get("model", ""):
+            eval_logger.warning(
+                "o1 models do not support `stop` and only support temperature=1"
+            )
         super().__init__(
             base_url=base_url,
             tokenizer_backend=tokenizer_backend,
@@ -238,3 +245,37 @@ def loglikelihood(self, requests, **kwargs):
         raise NotImplementedError(
             "Loglikelihood (and therefore `multiple_choice`-type tasks) is not supported for chat completions as OpenAI does not provide prompt logprobs. See https://github.com/EleutherAI/lm-evaluation-harness/issues/942#issuecomment-1777836312 or https://github.com/EleutherAI/lm-evaluation-harness/issues/1196 for more background on this limitation."
         )
+
+    def _create_payload(
+        self,
+        messages: List[Dict],
+        generate=False,
+        gen_kwargs: dict = None,
+        seed=1234,
+        **kwargs,
+    ) -> dict:
+        assert (
+            type(messages) is not str
+        ), "chat-completions require the --apply_chat_template flag."
+        gen_kwargs.pop("do_sample", False)
+        if "max_tokens" in gen_kwargs:
+            max_tokens = gen_kwargs.pop("max_tokens")
+        else:
+            max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
+        temperature = gen_kwargs.pop("temperature", 0)
+        stop = gen_kwargs.pop("until", ["
"])
+        if not isinstance(stop, (list, tuple)):
+            stop = [stop]
+        output = {
+            "messages": messages,
+            "model": self.model,
+            "max_completion_tokens": max_tokens,
+            "temperature": temperature,
+            "stop": stop[:4],
+            "seed": seed,
+            **gen_kwargs,
+        }
+        if "o1" in self.model:
+            output.pop("stop")
+            output["temperature"] = 1
+        return output


Test patch:
diff --git a/tests/test_tasks.py b/tests/test_tasks.py
index 221b533d3e..e24109966e 100644
--- a/tests/test_tasks.py
+++ b/tests/test_tasks.py
@@ -1,6 +1,7 @@
 import os
 from itertools import islice
 
+import datasets
 import pytest
 
 import lm_eval.tasks as tasks
@@ -10,6 +11,7 @@
 from .utils import new_tasks
 
 
+datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 task_manager = tasks.TaskManager()
 # Default Task