Repository: nomic-ai/gpt4all
File: gpt4all-bindings/python/gpt4all/gpt4all.py (Python)

Code before:
            repeat_last_n: How far in the models generation history to apply the repeat penalty.
            n_batch: Number of prompt tokens processed in parallel. Larger values decrease latency but increase resource requirements.
            n_predict: Equivalent to max_tokens, exists for backwards compatibility.
            streaming: If True, this method will instead return a generator that yields tokens as the model generates them.
            callback: A function with arguments token_id:int and response:str, which receives the tokens from the model as they are generated and stops the generation by returning False.
            verbose: If True, prints out the final prompt before it is passed to the model.

        Returns:
            Either the entire completion or a generator that yields the completion token by token.
        """
        generate_kwargs = dict(
            temp=temp,
            top_k=top_k,
            top_p=top_p,
            repeat_penalty=repeat_penalty,
            repeat_last_n=repeat_last_n,
            n_batch=n_batch,
            n_predict=n_predict if n_predict is not None else max_tokens,
            verbose=verbose,
        )

        if self._is_chat_session_activated:
            self.current_chat_session.append({"role": "user", "content": prompt})
            prompt = self._format_chat_prompt_template(
                         messages = self.current_chat_session,
                         default_prompt_header = self.chat_session_header
                     )
            generate_kwargs['reset_context'] = len(self.current_chat_session) == 1
        else:
            generate_kwargs['reset_context'] = True
        
        output_collector = [{"content": ""}]

        if self._is_chat_session_activated:
            self.current_chat_session.append({"role": "assistant", "content": ""})
            output_collector = self.current_chat_session

        def _callback_wrapper(callback, output_collector):
            def _callback(token_id, response):
                nonlocal callback, output_collector

                output_collector[-1]["content"] += response
                
                return callback(token_id, response)

            return _callback
        
        if streaming:
            return self.model.prompt_model_streaming(prompt = prompt, callback = _callback_wrapper(callback, output_collector), **generate_kwargs)

        self.model.prompt_model(prompt = prompt, callback = _callback_wrapper(callback, output_collector), **generate_kwargs)

Review comment (suggestion):
Not sure, but isn't the `chat_session_header` supposed to be inserted only at the start of a session? This looks like it goes in there every time a new prompt is started.

Code after:
                return url
            return f"https://gpt4all.io/models/{model_filename}"

        # Download model
        download_path = os.path.join(model_path, model_filename).replace("\\", "\\\\")
        download_url = get_download_url(model_filename)

        response = requests.get(download_url, stream=True)
        total_size_in_bytes = int(response.headers.get("content-length", 0))
        block_size = 2**20  # 1 MB

        with tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar:
            try:
                with open(download_path, "wb") as file:
                    for data in response.iter_content(block_size):
                        progress_bar.update(len(data))
                        file.write(data)
            except Exception:
                if os.path.exists(download_path):
                    if verbose:
                        print("Cleaning up the interrupted download...")
                    os.remove(download_path)
                raise

        # Validate download was successful
        if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
            raise RuntimeError("An error occurred during download. Downloaded file may not work.")

        # Sleep for a little bit so OS can remove file lock
        time.sleep(2)

        if verbose:
            print("Model downloaded at: ", download_path)
        return download_path

    def generate(
        self,
        prompt: str,
        max_tokens: int = 200,
        temp: float = 0.7,
        top_k: int = 40,
        top_p: float = 0.4,
        repeat_penalty: float = 1.18,
        repeat_last_n: int = 64,
        n_batch: int = 8,
        n_predict: Optional[int] = None,
        streaming: bool = False,
        callback: pyllmodel.ResponseCallbackType = pyllmodel.empty_response_callback,
    ) -> Union[str, Iterable[str]]:
        """
        Generate outputs from any GPT4All model.