import inspect import os from functools import partial from typing import Dict, Any, Optional, List from langchain.callbacks.manager import CallbackManagerForLLMRun from pydantic import root_validator from langchain.llms import gpt4all from dotenv import dotenv_values from utils import FakeTokenizer def get_model_tokenizer_gpt4all(base_model, **kwargs): # defaults (some of these are generation parameters, so need to be passed in at generation time) model_kwargs = dict(n_threads=os.cpu_count() // 2, temp=kwargs.get('temperature', 0.2), top_p=kwargs.get('top_p', 0.75), top_k=kwargs.get('top_k', 40), n_ctx=2048 - 256) env_gpt4all_file = ".env_gpt4all" model_kwargs.update(dotenv_values(env_gpt4all_file)) # make int or float if can to satisfy types for class for k, v in model_kwargs.items(): try: if float(v) == int(v): model_kwargs[k] = int(v) else: model_kwargs[k] = float(v) except: pass if base_model == "llama": if 'model_path_llama' not in model_kwargs: raise ValueError("No model_path_llama in %s" % env_gpt4all_file) model_path = model_kwargs.pop('model_path_llama') # FIXME: GPT4All version of llama doesn't handle new quantization, so use llama_cpp_python from llama_cpp import Llama # llama sets some things at init model time, not generation time func_names = list(inspect.signature(Llama.__init__).parameters) model_kwargs = {k: v for k, v in model_kwargs.items() if k in func_names} model_kwargs['n_ctx'] = int(model_kwargs['n_ctx']) model = Llama(model_path=model_path, **model_kwargs) elif base_model in "gpt4all_llama": if 'model_name_gpt4all_llama' not in model_kwargs and 'model_path_gpt4all_llama' not in model_kwargs: raise ValueError("No model_name_gpt4all_llama or model_path_gpt4all_llama in %s" % env_gpt4all_file) model_name = model_kwargs.pop('model_name_gpt4all_llama') model_type = 'llama' from gpt4all import GPT4All as GPT4AllModel model = GPT4AllModel(model_name=model_name, model_type=model_type) elif base_model in "gptj": if 'model_name_gptj' not in model_kwargs and 'model_path_gptj' not in model_kwargs: raise ValueError("No model_name_gpt4j or model_path_gpt4j in %s" % env_gpt4all_file) model_name = model_kwargs.pop('model_name_gptj') model_type = 'gptj' from gpt4all import GPT4All as GPT4AllModel model = GPT4AllModel(model_name=model_name, model_type=model_type) else: raise ValueError("No such base_model %s" % base_model) return model, FakeTokenizer(), 'cpu' from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler class H2OStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler): def on_llm_new_token(self, token: str, **kwargs: Any) -> None: """Run on new LLM token. Only available when streaming is enabled.""" # streaming to std already occurs without this # sys.stdout.write(token) # sys.stdout.flush() pass def get_model_kwargs(env_kwargs, default_kwargs, cls, exclude_list=[]): # default from class model_kwargs = {k: v.default for k, v in dict(inspect.signature(cls).parameters).items() if k not in exclude_list} # from our defaults model_kwargs.update(default_kwargs) # from user defaults model_kwargs.update(env_kwargs) # ensure only valid keys func_names = list(inspect.signature(cls).parameters) model_kwargs = {k: v for k, v in model_kwargs.items() if k in func_names} return model_kwargs def get_llm_gpt4all(model_name, model=None, max_new_tokens=256, temperature=0.1, repetition_penalty=1.0, top_k=40, top_p=0.7, streaming=False, callbacks=None, prompter=None, context='', iinput='', verbose=False, ): assert prompter is not None env_gpt4all_file = ".env_gpt4all" env_kwargs = dotenv_values(env_gpt4all_file) max_tokens = env_kwargs.pop('max_tokens', 2048 - max_new_tokens) default_kwargs = dict(context_erase=0.5, n_batch=1, max_tokens=max_tokens, n_predict=max_new_tokens, repeat_last_n=64 if repetition_penalty != 1.0 else 0, repeat_penalty=repetition_penalty, temp=temperature, temperature=temperature, top_k=top_k, top_p=top_p, use_mlock=True, verbose=verbose) if model_name == 'llama': cls = H2OLlamaCpp model_path = env_kwargs.pop('model_path_llama') if model is None else model model_kwargs = get_model_kwargs(env_kwargs, default_kwargs, cls, exclude_list=['lc_kwargs']) model_kwargs.update(dict(model_path=model_path, callbacks=callbacks, streaming=streaming, prompter=prompter, context=context, iinput=iinput)) llm = cls(**model_kwargs) llm.client.verbose = verbose elif model_name == 'gpt4all_llama': cls = H2OGPT4All model_path = env_kwargs.pop('model_path_gpt4all_llama') if model is None else model model_kwargs = get_model_kwargs(env_kwargs, default_kwargs, cls, exclude_list=['lc_kwargs']) model_kwargs.update( dict(model=model_path, backend='llama', callbacks=callbacks, streaming=streaming, prompter=prompter, context=context, iinput=iinput)) llm = cls(**model_kwargs) elif model_name == 'gptj': cls = H2OGPT4All model_path = env_kwargs.pop('model_path_gptj') if model is None else model model_kwargs = get_model_kwargs(env_kwargs, default_kwargs, cls, exclude_list=['lc_kwargs']) model_kwargs.update( dict(model=model_path, backend='gptj', callbacks=callbacks, streaming=streaming, prompter=prompter, context=context, iinput=iinput)) llm = cls(**model_kwargs) else: raise RuntimeError("No such model_name %s" % model_name) return llm class H2OGPT4All(gpt4all.GPT4All): model: Any prompter: Any context: Any = '' iinput: Any = '' """Path to the pre-trained GPT4All model file.""" @root_validator() def validate_environment(cls, values: Dict) -> Dict: """Validate that the python package exists in the environment.""" try: if isinstance(values["model"], str): from gpt4all import GPT4All as GPT4AllModel full_path = values["model"] model_path, delimiter, model_name = full_path.rpartition("/") model_path += delimiter values["client"] = GPT4AllModel( model_name=model_name, model_path=model_path or None, model_type=values["backend"], allow_download=False, ) if values["n_threads"] is not None: # set n_threads values["client"].model.set_thread_count(values["n_threads"]) else: values["client"] = values["model"] try: values["backend"] = values["client"].model_type except AttributeError: # The below is for compatibility with GPT4All Python bindings <= 0.2.3. values["backend"] = values["client"].model.model_type except ImportError: raise ValueError( "Could not import gpt4all python package. " "Please install it with `pip install gpt4all`." ) return values def _call( self, prompt: str, stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs, ) -> str: # Roughly 4 chars per token if natural language n_ctx = 2048 prompt = prompt[-self.max_tokens * 4:] # use instruct prompting data_point = dict(context=self.context, instruction=prompt, input=self.iinput) prompt = self.prompter.generate_prompt(data_point) verbose = False if verbose: print("_call prompt: %s" % prompt, flush=True) # FIXME: GPT4ALl doesn't support yield during generate, so cannot support streaming except via itself to stdout return super()._call(prompt, stop=stop, run_manager=run_manager) from langchain.llms import LlamaCpp class H2OLlamaCpp(LlamaCpp): model_path: Any prompter: Any context: Any iinput: Any """Path to the pre-trained GPT4All model file.""" @root_validator() def validate_environment(cls, values: Dict) -> Dict: """Validate that llama-cpp-python library is installed.""" if isinstance(values["model_path"], str): model_path = values["model_path"] model_param_names = [ "lora_path", "lora_base", "n_ctx", "n_parts", "seed", "f16_kv", "logits_all", "vocab_only", "use_mlock", "n_threads", "n_batch", "use_mmap", "last_n_tokens_size", ] model_params = {k: values[k] for k in model_param_names} # For backwards compatibility, only include if non-null. if values["n_gpu_layers"] is not None: model_params["n_gpu_layers"] = values["n_gpu_layers"] try: from llama_cpp import Llama values["client"] = Llama(model_path, **model_params) except ImportError: raise ModuleNotFoundError( "Could not import llama-cpp-python library. " "Please install the llama-cpp-python library to " "use this embedding model: pip install llama-cpp-python" ) except Exception as e: raise ValueError( f"Could not load Llama model from path: {model_path}. " f"Received error {e}" ) else: values["client"] = values["model_path"] return values def _call( self, prompt: str, stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs, ) -> str: verbose = False # tokenize twice, just to count tokens, since llama cpp python wrapper has no way to truncate # still have to avoid crazy sizes, else hit llama_tokenize: too many tokens -- might still hit, not fatal prompt = prompt[-self.n_ctx * 4:] prompt_tokens = self.client.tokenize(b" " + prompt.encode("utf-8")) num_prompt_tokens = len(prompt_tokens) if num_prompt_tokens > self.n_ctx: # conservative by using int() chars_per_token = int(len(prompt) / num_prompt_tokens) prompt = prompt[-self.n_ctx * chars_per_token:] if verbose: print("reducing tokens, assuming average of %s chars/token: %s" % chars_per_token, flush=True) prompt_tokens2 = self.client.tokenize(b" " + prompt.encode("utf-8")) num_prompt_tokens2 = len(prompt_tokens2) print("reduced tokens from %d -> %d" % (num_prompt_tokens, num_prompt_tokens2), flush=True) # use instruct prompting data_point = dict(context=self.context, instruction=prompt, input=self.iinput) prompt = self.prompter.generate_prompt(data_point) if verbose: print("_call prompt: %s" % prompt, flush=True) if self.streaming: text_callback = None if run_manager: text_callback = partial( run_manager.on_llm_new_token, verbose=self.verbose ) # parent handler of streamer expects to see prompt first else output="" and lose if prompt=None in prompter if text_callback: text_callback(prompt) text = "" for token in self.stream(prompt=prompt, stop=stop, run_manager=run_manager): text_chunk = token["choices"][0]["text"] # self.stream already calls text_callback # if text_callback: # text_callback(text_chunk) text += text_chunk return text else: params = self._get_parameters(stop) params = {**params, **kwargs} result = self.client(prompt=prompt, **params) return result["choices"][0]["text"]