Repository: nomic-ai/gpt4all
File: gpt4all-bindings/python/gpt4all/pyllmodel.py (Python)

Code before:
        ----------
        prompt: str
            Question, task, or conversation for model to respond to
        callback(token_id:int, response:str): bool
            The model sends response tokens to callback

        Returns
        -------
        None
        """

        logging.info("pyllmodel.py: LLModel.prompt_model -- prompt:\n" +
                     "%s\n" +
                     "===/pyllmodel.py: LLModel.prompt_model -- prompt/===", prompt)


        prompt_bytes = prompt.encode('utf-8')
        prompt_ptr = ctypes.c_char_p(prompt_bytes)

        def _callback_wrapper(callback: ResponseCallbackType) -> ResponseCallbackType:
            
            def _callback(token_id: int, response: str) -> bool:
                nonlocal callback
                return callback(token_id, response.decode('utf-8', 'replace'))

            return _callback
            
        self._set_context(
            n_predict=n_predict,
            top_k=top_k,
            top_p=top_p,
            temp=temp,
            n_batch=n_batch,
            repeat_penalty=repeat_penalty,
            repeat_last_n=repeat_last_n,
            context_erase=context_erase,
            reset_context=reset_context,
        )

        llmodel.llmodel_prompt(
            self.model,
            prompt_ptr,
            PromptCallback(self._prompt_callback),
            ResponseCallback( _callback_wrapper( callback ) ),
            RecalculateCallback(self._recalculate_callback),
            self.context,
        )

    def prompt_model_streaming(self, 
        prompt: str, 
        callback: ResponseCallbackType = empty_response_callback, 

Review comment (suggestion):
And here instead of:
```py
        def _callback_wrapper(callback: ResponseCallbackType) -> ResponseCallbackType:
            
            def _callback(token_id: int, response: str) -> bool:
                nonlocal callback
                return callback(token_id, response.decode('utf-8', 'replace'))

            return _callback
```
it should be:
```py
        def _callback_wrapper(callback: ResponseCallbackType) -> RawResponseCallbackType:
            
            def _callback(token_id: int, response: bytes) -> bool:
                nonlocal callback
                return callback(token_id, response.decode('utf-8', 'replace'))

            return _callback
```
Then as explained:
- please don't nest it inside `prompt_model()`.
- better naming would be neat.

Code after:
        self.context.context_erase = context_erase

    def generate_embedding(
        self,
        text: str
    ) -> List[float]:
        if not text:
            raise ValueError("Text must not be None or empty")

        embedding_size = ctypes.c_size_t()
        c_text = ctypes.c_char_p(text.encode('utf-8'))
        embedding_ptr = llmodel.llmodel_embedding(self.model, c_text, ctypes.byref(embedding_size))
        embedding_array = [embedding_ptr[i] for i in range(embedding_size.value)]
        llmodel.llmodel_free_embedding(embedding_ptr)
        return list(embedding_array)

    def prompt_model(
        self,
        prompt: str,
        callback: ResponseCallbackType,
        n_predict: int = 4096,
        top_k: int = 40,
        top_p: float = 0.9,
        temp: float = 0.1,
        n_batch: int = 8,
        repeat_penalty: float = 1.2,
        repeat_last_n: int = 10,
        context_erase: float = 0.75,
        reset_context: bool = False,
    ):
        """
        Generate response from model from a prompt.

        Parameters
        ----------
        prompt: str
            Question, task, or conversation for model to respond to
        callback(token_id:int, response:str): bool
            The model sends response tokens to callback

        Returns
        -------
        None
        """

        logger.info(
            "LLModel.prompt_model -- prompt:\n"
            + "%s\n"
            + "===/LLModel.prompt_model -- prompt/===",
            prompt,
        )