Repository: ggerganov/llama.cpp
File: scripts/server-bench.py (Python)

Code before:
    session = data["session"]
    server_address: str = data["server_address"]

    t_submit = time()
    if data["external_server"]:
        json_data: dict = {
            "prompt": data["prompt"], "ignore_eos": True,
            "seed": data["seed"], "max_tokens": data["n_predict"], "stream": True}
        response = session.post(f"{server_address}/v1/completions", json=json_data, stream=True)
    elif data["synthetic_prompt"]:
        json_data: dict = {
            "prompt": data["prompt"], "ignore_eos": True, "cache_prompt": False,
            "seed": data["seed"], "n_predict": data["n_predict"], "stream": True}
        response = session.post(f"{server_address}/completion", json=json_data, stream=True)
    else:
        response = session.post(
            f"{server_address}/apply-template",
            json={"messages": [{"role": "user", "content": data["prompt"], "stream": True}]}
        )
        if response.status_code != 200:
            response_text = ""
            try:
                response_text = response.text
                response_text = ": {response_text}"
            except RuntimeError:
                pass
            raise RuntimeError(f"Server returned status code {response.status_code}{response_text}")
        prompt: str = json.loads(response.text)["prompt"]

        json_data: dict = {"prompt": prompt, "seed": data["seed"], "n_predict": data["n_predict"], "stream": True}
        response = session.post(f"{server_address}/completion", json=json_data, stream=True)

    lines = []
    token_arrival_times: list[float] = []
    for line in response.iter_lines(decode_unicode=False):
        if not line.startswith(b"data: "):
            continue
        lines.append(line)
        token_arrival_times.append(time())
    token_arrival_times = token_arrival_times[:-1]
    if "timings" in json.loads(lines[-2][6:]):
        token_arrival_times = token_arrival_times[:-1]

    if response.status_code != 200:
        response_text = ""
        try:
            response_text = response.text
            response_text = ": {response_text}"
        except RuntimeError:
            pass
        raise RuntimeError(f"Server returned status code {response.status_code}{response_text}")

Review comment (suggestion):
I don't understand? Did you mean to just do this:
```python
response_text = f": {response.text}"
```

Code after:

    t_submit = time()
    if data["external_server"]:
        json_data: dict = {
            "prompt": data["prompt"], "ignore_eos": True,
            "seed": data["seed"], "max_tokens": data["n_predict"], "stream": True}
        response = session.post(f"{server_address}/v1/completions", json=json_data, stream=True)
    elif data["synthetic_prompt"]:
        json_data: dict = {
            "prompt": data["prompt"], "ignore_eos": True, "cache_prompt": False,
            "seed": data["seed"], "n_predict": data["n_predict"], "stream": True}
        response = session.post(f"{server_address}/completion", json=json_data, stream=True)
    else:
        response = session.post(
            f"{server_address}/apply-template",
            json={"messages": [{"role": "user", "content": data["prompt"], "stream": True}]}
        )
        response.raise_for_status()
        prompt: str = json.loads(response.text)["prompt"]

        json_data: dict = {"prompt": prompt, "seed": data["seed"], "n_predict": data["n_predict"], "stream": True}
        response = session.post(f"{server_address}/completion", json=json_data, stream=True)
    response.raise_for_status()

    lines = []
    token_arrival_times: list[float] = []
    for line in response.iter_lines(decode_unicode=False):
        if not line.startswith(b"data: "):
            continue
        lines.append(line)
        token_arrival_times.append(time())
    token_arrival_times = token_arrival_times[:-1]
    if len(lines) > 1 and "timings" in json.loads(lines[-2][6:]):
        token_arrival_times = token_arrival_times[:-1]

    return (t_submit, token_arrival_times)


def benchmark(
        path_server: str, path_log: Optional[str], path_db: Optional[str], name: Optional[str], prompt_source: str, n_prompts: int,
        n_predict: int, n_predict_min: int, seed_offset: int):
    external_server: bool = path_server.startswith("http://") or path_server.startswith("https://")
    if os.environ.get("LLAMA_ARG_N_PARALLEL") is None:
        logger.info("LLAMA_ARG_N_PARALLEL not explicitly set, using 32")
        os.environ["LLAMA_ARG_N_PARALLEL"] = "32"
    if not external_server and os.environ.get("LLAMA_ARG_N_GPU_LAYERS") is None:
        logger.info("LLAMA_ARG_N_GPU_LAYERS not explicitly set, using 999")
        os.environ["LLAMA_ARG_N_GPU_LAYERS"] = "999"
    if not external_server and os.environ.get("LLAMA_ARG_FLASH_ATTN") is None:
        logger.info("LLAMA_ARG_FLASH_ATTN not explicitly set, using 'true'")
        os.environ["LLAMA_ARG_FLASH_ATTN"] = "true"