utils/truncate.py

import tiktoken

encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")


def truncate_string(s, max_length=1024) -> str:
    e = encoding.encode(s)[:max_length]
    s = encoding.decode(e)
    return s