Repository: google/gemma.cpp
File: gemma/run.cc (C++)

Code before:
      // never part of the dialog, see Table 5 in the Gemma-2 paper:
      // https://arxiv.org/pdf/2408.00118
      // Or we have hit max_generated_tokens, then the last token will be lost.
      // (We could store it in stream_token, and then prepend to the next turn,
      // but it's not worth the complexity, as multi-turn with max_generated is
      // not a common use case.)
      // In either case, we need to rewind abs_pos by one.
      HWY_ASSERT(abs_pos > 0);
      abs_pos--;
    }
  }
}

void Run(ThreadingArgs& threading, LoaderArgs& loader,
         InferenceArgs& inference) {
  PROFILER_ZONE("Run.misc");

  // Note that num_threads is an upper bound; we also limit to the number of
  // detected and enabled cores.
  MatMulEnv env(MakeMatMulEnv(threading));
  if (inference.verbosity >= 2) env.print_best = true;
  Gemma model = CreateGemma(loader, env);
  KVCache kv_cache =
      KVCache::Create(model.GetModelConfig(), inference.prefill_tbatch_size);

  if (!threading.prompt.empty()) {
    std::vector<int> prompt =
        WrapAndTokenize(model.Tokenizer(), model.ChatTemplate(), model.Info(),
                        0, threading.prompt);

    TimingInfo timing_info = {.verbosity = inference.verbosity};
    RuntimeConfig runtime_config = {.gen = nullptr,  // Use default generator
                                    .verbosity = inference.verbosity,
                                    .use_spinning = threading.spin};
    inference.CopyTo(runtime_config);

    model.Generate(runtime_config, prompt, 0, 0, kv_cache, timing_info);
    std::cout << "\n";
    return;  // Exit after generating response
  }

  if (inference.verbosity >= 1) {
    std::string instructions =
        "*Usage*\n"
        "  Enter an instruction and press enter (%C resets conversation, "
        "%Q quits).\n";
    const std::string multiturn =
        inference.multiturn == 0
            ? std::string(
                  "  Since multiturn is set to 0, conversation will "
                  "automatically reset every turn.\n\n")

Review comment (refactor):
I'd prefer not to duplicate this code, which increases maintenance burden. Instead we can re-use the existing ReplGemma, and pass to GetPrompt the inference args (rather than individual fields), and check inside GetPrompt if prompt is not empty, and if so return that immediately.
Then to exit, we could introduce a flag outside of the `while(true)` loop that checks whether prompt was not empty, and if so, exit the loop. This adds another condition next to the existing `prompt_string[1] == 'q'`. What do you think?

Code after:
    }

    // Prepare for the next turn. Works only for PaliGemma.
    if (!inference.multiturn ||
        model.Info().wrapping == PromptWrapping::PALIGEMMA) {
      abs_pos = 0;  // Start a new turn at position 0.
      InitGenerator(inference, gen);
    } else {
      // The last token was either EOS, then it should be ignored because it is
      // never part of the dialog, see Table 5 in the Gemma-2 paper:
      // https://arxiv.org/pdf/2408.00118
      // Or we have hit max_generated_tokens, then the last token will be lost.
      // (We could store it in stream_token, and then prepend to the next turn,
      // but it's not worth the complexity, as multi-turn with max_generated is
      // not a common use case.)
      // In either case, we need to rewind abs_pos by one.
      HWY_ASSERT(abs_pos > 0);
      abs_pos--;
    }
  }
}

void Run(ThreadingArgs& threading, LoaderArgs& loader,
         InferenceArgs& inference) {
  PROFILER_ZONE("Run.misc");

  // Note that num_threads is an upper bound; we also limit to the number of
  // detected and enabled cores.
  MatMulEnv env(MakeMatMulEnv(threading));
  if (inference.verbosity >= 2) env.print_best = true;
  Gemma model = CreateGemma(loader, env);
  KVCache kv_cache =
      KVCache::Create(model.GetModelConfig(), inference.prefill_tbatch_size);

  if (inference.verbosity >= 1) {
    std::string instructions =
        "*Usage*\n"
        "  Enter an instruction and press enter (%C resets conversation, "
        "%Q quits).\n";
    const std::string multiturn =
        inference.multiturn == 0
            ? std::string(
                  "  Since multiturn is set to 0, conversation will "
                  "automatically reset every turn.\n\n")
            : "\n";
    const std::string examples =
        "*Examples*\n"
        "  - Write an email to grandma thanking her for the cookies.\n"
        "  - What are some historical attractions to visit around "
        "Massachusetts?\n"
        "  - Compute the nth fibonacci number in javascript.\n"