Repository: evalplus/evalplus
Issue: 🤗 [REQUEST] - Gemini Pro
Root cause: Unimplemented Feature Gap

### Model introduction

Gemini Pro by google: https://blog.google/technology/ai/google-gemini-ai/

### Model URL

https://cloud.google.com/blog/products/ai-machine-learning/gemini-support-on-vertex-ai

### Data type

None of above: specify the details in the 'Other context' section

### Additional instructions (Optional)

_No response_

### Author

No

### Security

- [ ] I confirm that the model is safe to run which does not contain any malicious code or content.

### Integrity

- [ ] I confirm that the model comes from unique and original work and does not contain any plagiarism.

``````python
import vertexai
import asyncio
import tenacity
import os
import argparse
import time
from evalplus.data import get_human_eval_plus, get_mbpp_plus, write_jsonl
from google.api_core.exceptions import InternalServerError, ResourceExhausted
from vertexai.preview.generative_models import GenerativeModel
from typing import Sequence, Any, TypeVar, Iterable

_T = TypeVar("_T")


def chunked(seq: Sequence[_T], n: int) -> Iterable[Sequence[_T]]:
    """Yield successive n-sized chunks from seq."""
    return (seq[i : i + n] for i in range(0, len(seq), n))


def retry(errors: Any, max_attempts: int = 5):
    return tenacity.retry(
        retry=tenacity.retry_if_exception_type(errors),
        wait=tenacity.wait_exponential(multiplier=1, min=5, max=20),
        stop=tenacity.stop_after_attempt(max_attempts),
        before_sleep=print,
    )


PROJECT_ID = os.getenv("GOOGLE_API_PROJECT_ID")
LOCATION = os.getenv("GOOGLE_API_LOCATION", "us-central1")


@retry((IndexError, ResourceExhausted, InternalServerError))
async def gemini_chat_completions_with_backoff(
    model_name: str,
    messages: list[dict[str, str]],
):
    assert model_name == "gemini-pro"
    model = GenerativeModel(model_name)
    contents = [
        {"role": message["role"], "parts": [{"text": message["content"]}]}
        for message in messages
    ]
    response = await model.generate_content_async(
        contents,
        generation_config={
            "temperature": 0.0,
            "top_p": 1.0,
        },
        safety_settings={0: 0, 1: 0, 2: 0, 3: 0, 4: 0},
    )
    candidate = response.candidates[0]
    return candidate.text


PROMPT_TEMPLATE = """Please generate code to complete the following problem. 
```python
{prompt}
```
Enclose your solution in triple backticks."""


async def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--path", type=str)
    parser.add_argument("--dataset", type=str, choices=["humaneval", "mbpp"])
    parser.add_argument(
        "--rpm", type=int, default=45, help="Requests per minute to send to the API"
    )

    vertexai.init(project=PROJECT_ID, location=LOCATION)

    args = parser.parse_args()
    problems = get_human_eval_plus() if args.dataset == "humaneval" else get_mbpp_plus()

    batches = chunked(list(problems.values()), n=args.rpm)
    for index, batch in enumerate(batches):
        if index != 0:
            print("Sleeping for 60 seconds")
            time.sleep(60)
        print(f"Batch {index}")
        tasks = list[Any]()
        for problem in batch:
            raw_prompt = problem["prompt"].strip()
            prompt = PROMPT_TEMPLATE.format(prompt=raw_prompt)
            messages = [
                {"role": "user", "content": prompt},
            ]
            tasks.append(gemini_chat_completions_with_backoff("gemini-pro", messages))
        task_results = await asyncio.gather(*tasks, return_exceptions=True)
        results = list[dict]()
        for problem, result in zip(batch, task_results):
            if isinstance(result, Exception):
                print("Error in processing", problem["task_id"])
                completion = ":clown:"
            else:
                completion = result
            # extract the code
            raw_prompt = problem["prompt"].strip()
            try:
                completion = completion.split("```")[1]
                if completion.startswith("python"):
                    completion = completion[6:]
                solution = f"{raw_prompt}\n{completion}"
                results.append({"task_id": problem["task_id"], "solution": solution})
            except IndexError:
                results.append(
                    {"task_id": problem["task_id"], "completion": completion}
                )
        write_jsonl(args.path, results, append=True)


asyncio.run(main())
``````

Unofficial results with the code above:
```
humaneval (base tests)
pass@1: 0.634
humaneval+ (base + extra tests)
pass@1: 0.555
```

@UniverseFly Thanks! Would you also help evaluate the MBPP scores and I can update them soon.

@ganler Code updated above.
```
mbpp (base tests)
pass@1: 0.729
mbpp+ (base + extra tests)
pass@1: 0.579
```

Thanks! The leaderboard is updated.