Repository: LAION-AI/Open-Assistant
File: inference/worker/utils.py (Python)

Code before:
        ids = ids[-(max_input_length - 1) :]
        with shared_tokenizer_lock:
            prompt = tokenizer.decode(ids)
            prompt = add_prefix_to_prompt(prompt)

    input_length = len(ids)
    spare = max_total_tokens - input_length - 1
    if not parameters.max_new_tokens:
        parameters.max_new_tokens = spare
    elif parameters.max_new_tokens > spare:
        logger.debug(f"Max new tokens too high, reducing to {spare}")
        parameters.max_new_tokens = spare
    return prompt


def wait_for_inference_server(http: "HttpClient", timeout: int = 600):
    time_limit = time.time() + timeout
    while True:
        try:
            response = http.get("/health")
            response.raise_for_status()
        except (requests.HTTPError, requests.ConnectionError):
            if time.time() > time_limit:
                raise
            sleep_duration = random.uniform(0, 10)
            logger.debug(f"Inference server not ready. Retrying in {sleep_duration:.2f} seconds")
            time.sleep(sleep_duration)
        else:
            logger.info("Inference server is ready")
            break


def text_to_events(text: str, seed: int | None = None, pause: float = 0.0):
    tokens = text.split()
    for token in tokens[:-1]:
        yield interface.GenerateStreamResponse(
            token=interface.Token(
                text=token + " ",
                logprob=0.1,
                id=0,
            ),
        )
        if pause > 0:
            time.sleep(pause)
    yield interface.GenerateStreamResponse(
        token=interface.Token(
            text=tokens[-1],
            logprob=0.1,
            id=0,
        ),
        generated_text=text,

Review comment (bug):
I think this one should remain `warning`, it's relevant outside of debugging.

Code after:
                ids = tokenizer.encode(V2_PROMPTER_PREFIX) + ids

    max_total_tokens = worker_config.model_config.max_total_length
    input_length = len(ids)
    spare = max_total_tokens - input_length - 1

    if not parameters.max_new_tokens:
        parameters.max_new_tokens = spare
    elif parameters.max_new_tokens > spare:
        logger.debug(f"Max new tokens too high, reducing to {spare}")
        parameters.max_new_tokens = spare

    return prompt


def wait_for_inference_server(http: "HttpClient", timeout: int = 600):
    time_limit = time.time() + timeout
    while True:
        try:
            response = http.get("/health")
            response.raise_for_status()
        except (requests.HTTPError, requests.ConnectionError):
            if time.time() > time_limit:
                raise
            sleep_duration = random.uniform(0, 10)
            logger.warning(f"Inference server not ready. Retrying in {sleep_duration:.2f} seconds")
            time.sleep(sleep_duration)
        else:
            logger.info("Inference server is ready")
            break


def text_to_events(text: str, seed: int | None = None, pause: float = 0.0):
    tokens = text.split()
    for token in tokens[:-1]:
        yield interface.GenerateStreamResponse(
            token=interface.Token(
                text=token + " ",
                logprob=0.1,
                id=0,
            ),
        )
        if pause > 0:
            time.sleep(pause)
    yield interface.GenerateStreamResponse(
        token=interface.Token(
            text=tokens[-1],
            logprob=0.1,
            id=0,
        ),
        generated_text=text,