Repository: nomic-ai/gpt4all
File: gpt4all-backend/src/llmodel_shared.cpp (C++)

Code before:
                           bool allowContextShift,
                           PromptContext &promptCtx,
                           std::vector<Token> embd_inp,
                           bool isResponse,
                           bool alwaysDecode) {
    if ((int) embd_inp.size() > contextLength() - 4) {
        // FIXME: (Adam) We should find a way to bubble these strings to the UI level to allow for
        // translation
        responseCallback(-1, "Your message was too long and could not be processed. Please try again with something shorter.");
        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
            " tokens and the context window is " << contextLength() << "!\n";
        return false;
    }

    // FIXME(jared): There are mitigations for this situation, such as making room before
    // copying the prompt context, or restoring the KV cache when we restore the prompt
    // context.
    if (!allowContextShift && promptCtx.n_past + embd_inp.size() > contextLength()) {
        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_eval=" << embd_inp.size()
                  << ", n_ctx=" << contextLength() << "\n";
        return false;
    }

    // always decode something before generating, even if cached
    if (alwaysDecode && embd_inp.empty()) {
        auto cache = inputTokens();
        if (cache.empty())
            throw std::runtime_error("zero token prompt is not supported");
        embd_inp.push_back(cache.back());
        promptCtx.n_past--;
    }

    // Find the greatest n_past where the beginning of embd_inp matches the end of the token cache, starting at the
    // requested n_past.
    // This is used to skip unnecessary work when the prompt shares a common prefix with the previous result.
    auto embd_inp_start = computeModelInputPosition(promptCtx, embd_inp);
    size_t start_offset = embd_inp_start - embd_inp.begin();

    // always decode up to a full batch before generating, even if cached
    if (alwaysDecode)
        start_offset -= std::min(promptCtx.n_batch, int32_t(start_offset));

    setModelInputPosition(promptCtx, promptCtx.n_past + start_offset);

    // execute the callback even for skipped tokens
    size_t i = 0;
    for (; i < start_offset; i++) {
        Token tok = embd_inp[i];
        bool res = isResponse ? responseCallback(tok, tokenToString(tok)) : promptCallback(tok);
        if (!res)
            return false;

Review comment (bug):
llmodel.h has this:

```
    // Use only for testing and debugging. Throws if the backend was built with -DNDEBUG.
    virtual std::span<const Token> inputTokens() const = 0;
```

Code after:
                           bool allowContextShift,
                           PromptContext &promptCtx,
                           std::vector<Token> embd_inp,
                           bool isResponse,
                           bool alwaysDecode) {
    if ((int) embd_inp.size() > contextLength() - 4) {
        // FIXME: (Adam) We should find a way to bubble these strings to the UI level to allow for
        // translation
        responseCallback(-1, "Your message was too long and could not be processed. Please try again with something shorter.");
        std::cerr << implementation().modelType() << " ERROR: The prompt is " << embd_inp.size() <<
            " tokens and the context window is " << contextLength() << "!\n";
        return false;
    }

    // FIXME(jared): There are mitigations for this situation, such as making room before
    // copying the prompt context, or restoring the KV cache when we restore the prompt
    // context.
    if (!allowContextShift && promptCtx.n_past + embd_inp.size() > contextLength()) {
        std::cerr << "LLModel Warning: Not enough space, n_past=" << promptCtx.n_past << ", n_eval=" << embd_inp.size()
                  << ", n_ctx=" << contextLength() << "\n";
        return false;
    }

    // always decode something before generating, even if cached
    if (alwaysDecode && embd_inp.empty()) {
        auto cache = inputTokens();
        if (!promptCtx.n_past)
            throw std::runtime_error("zero token prompt is not supported");
        assert(!cache.empty());
        embd_inp.push_back(cache.back());
        promptCtx.n_past--;
    }

    // Find the greatest n_past where the beginning of embd_inp matches the end of the token cache, starting at the
    // requested n_past.
    // This is used to skip unnecessary work when the prompt shares a common prefix with the previous result.
    auto embd_inp_start = computeModelInputPosition(promptCtx, embd_inp);
    size_t start_offset = embd_inp_start - embd_inp.begin();

    // always decode up to a full batch before generating, even if cached
    if (alwaysDecode)
        start_offset -= std::min(promptCtx.n_batch, int32_t(start_offset));

    setModelInputPosition(promptCtx, promptCtx.n_past + start_offset);

    // execute the callback even for skipped tokens
    size_t i = 0;
    for (; i < start_offset; i++) {
        Token tok = embd_inp[i];
        bool res = isResponse ? responseCallback(tok, tokenToString(tok)) : promptCallback(tok);
        if (!res)