Repository: nomic-ai/gpt4all
File: gpt4all-bindings/python/gpt4all/gpt4all.py (Python)

Code before:
            def _callback(token_id, response):
                nonlocal callback, output_collector

                output_collector[-1]["content"] += response
                
                return callback(token_id, response)

            return _callback
        
        if streaming:
            return self.model.prompt_model_streaming(prompt = prompt, callback = _callback_wrapper(callback, output_collector), **generate_kwargs)

        self.model.prompt_model(prompt = prompt, callback = _callback_wrapper(callback, output_collector), **generate_kwargs)

        return output_collector[-1]["content"]

    @contextmanager
    def chat_session(self, header: str = ""):
        '''
        Context manager to hold an inference optimized chat session with a GPT4All model.

        Args:
            header: An initial instruction. It is inserted before the transcript of the conversation.
        '''
        # Code to acquire resource, e.g.:
        self._is_chat_session_activated = True
        self.current_chat_session = []
        self.chat_session_header = header
        try:
            yield self
        finally:
            # Code to release resource, e.g.:
            self._is_chat_session_activated = False
            self.current_chat_session = []
            self.chat_session_header = ""

    def _format_chat_prompt_template(
        self, messages: List[Dict], default_prompt_header: str = "", default_prompt_footer: str = ""
    ) -> str:
        """
        Helper method for building a prompt using template from list of messages.

        Args:
            messages:  List of dictionaries. Each dictionary should have a "role" key
                with value of "system", "assistant", or "user" and a "content" key with a
                string value. Messages are organized such that "system" messages are at top of prompt,
                and "user" and "assistant" messages are displayed in order. Assistant messages get formatted as
                "Response: {content}".

        Returns:
            Formatted prompt.

Review comment (bug):
Correct me if I'm wrong, by the way:  unless I've missed something in the new `LLModel` logic, response should be bytes.

Code after:
        max_tokens: int = 200,
        temp: float = 0.7,
        top_k: int = 40,
        top_p: float = 0.4,
        repeat_penalty: float = 1.18,
        repeat_last_n: int = 64,
        n_batch: int = 8,
        n_predict: Optional[int] = None,
        streaming: bool = False,
        callback: pyllmodel.ResponseCallbackType = pyllmodel.empty_response_callback,
    ) -> Union[str, Iterable[str]]:
        """
        Generate outputs from any GPT4All model.

        Args:
            prompt: The prompt for the model the complete.
            max_tokens: The maximum number of tokens to generate.
            temp: The model temperature. Larger values increase creativity but decrease factuality.
            top_k: Randomly sample from the top_k most likely tokens at each generation step. Set this to 1 for greedy decoding.
            top_p: Randomly sample at each generation step from the top most likely tokens whose probabilities add up to top_p.
            repeat_penalty: Penalize the model for repetition. Higher values result in less repetition.
            repeat_last_n: How far in the models generation history to apply the repeat penalty.
            n_batch: Number of prompt tokens processed in parallel. Larger values decrease latency but increase resource requirements.
            n_predict: Equivalent to max_tokens, exists for backwards compatibility.
            streaming: If True, this method will instead return a generator that yields tokens as the model generates them.
            callback: A function with arguments token_id:int and response:str, which receives the tokens from the model as they are generated and stops the generation by returning False.

        Returns:
            Either the entire completion or a generator that yields the completion token by token.
        """

        # Preparing the model request
        generate_kwargs: Dict[str, Any] = dict(
            temp=temp,
            top_k=top_k,
            top_p=top_p,
            repeat_penalty=repeat_penalty,
            repeat_last_n=repeat_last_n,
            n_batch=n_batch,
            n_predict=n_predict if n_predict is not None else max_tokens,
        )

        if self._is_chat_session_activated:
            generate_kwargs["reset_context"] = len(self.current_chat_session) == 1 # check if there is only one message, i.e. system prompt
            self.current_chat_session.append({"role": "user", "content": prompt})

            prompt = self._format_chat_prompt_template(
                messages = self.current_chat_session[-1:],
                default_prompt_header = self.current_chat_session[0]["content"] if generate_kwargs["reset_context"] else "",
            )
        else: