viruPVS/langchain-docs huggingface dataset:
input_body = {**model_kwargs}
        if provider == "anthropic":
            input_body["prompt"] = _human_assistant_format(prompt)
        elif provider in ("ai21", "cohere", "meta"):
            input_body["prompt"] = prompt
        elif provider == "amazon":
            input_body = dict()
            input_body["inputText"] = prompt
            input_body["textGenerationConfig"] = {**model_kwargs}
        else:
            input_body["inputText"] = prompt
        if provider == "anthropic" and "max_tokens_to_sample" not in input_body:
            input_body["max_tokens_to_sample"] = 256
        return input_body
[docs]    @classmethod
    def prepare_output(cls, provider: str, response: Any) -> str:
        if provider == "anthropic":
            response_body = json.loads(response.get("body").read().decode())
            return response_body.get("completion")
        else:
            response_body = json.loads(response.get("body").read())
        if provider == "ai21":
            return response_body.get("completions")[0].get("data").get("text")
        elif provider == "cohere":
            return response_body.get("generations")[0].get("text")
        elif provider == "meta":
            return response_body.get("generation")
        else:
            return response_body.get("results")[0].get("outputText")
[docs]    @classmethod
    def prepare_output_stream(
        cls, provider: str, response: Any, stop: Optional[List[str]] = None
    ) -> Iterator[GenerationChunk]:
        stream = response.get("body")
        if not stream:
            return
        if provider not in cls.provider_to_output_key_map:
            raise ValueError(
if provider not in cls.provider_to_output_key_map:
            raise ValueError(
                f"Unknown streaming response output key for provider: {provider}"
            )
        for event in stream:
            chunk = event.get("chunk")
            if chunk:
                chunk_obj = json.loads(chunk.get("bytes").decode())
                if provider == "cohere" and (
                    chunk_obj["is_finished"]
                    or chunk_obj[cls.provider_to_output_key_map[provider]]
                    == "<EOS_TOKEN>"
                ):
                    return
                # chunk obj format varies with provider
                yield GenerationChunk(
                    text=chunk_obj[cls.provider_to_output_key_map[provider]]
                )
[docs]class BedrockBase(BaseModel, ABC):
    """Base class for Bedrock models."""
    client: Any = Field(exclude=True)  #: :meta private:
    region_name: Optional[str] = None
    """The aws region e.g., `us-west-2`. Fallsback to AWS_DEFAULT_REGION env variable
    or region specified in ~/.aws/config in case it is not provided here.
    """
    credentials_profile_name: Optional[str] = Field(default=None, exclude=True)
    """The name of the profile in the ~/.aws/credentials or ~/.aws/config files, which
    has either access keys or role information specified.
    If not specified, the default credential profile or, if on an EC2 instance,
    credentials from IMDS will be used.
    See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    """
    model_id: str
    """Id of the model to call, e.g., amazon.titan-text-express-v1, this is
    equivalent to the modelId property in the list-foundation-models api"""
equivalent to the modelId property in the list-foundation-models api"""
    model_kwargs: Optional[Dict] = None
    """Keyword arguments to pass to the model."""
    endpoint_url: Optional[str] = None
    """Needed if you don't want to default to us-east-1 endpoint"""
    streaming: bool = False
    """Whether to stream the results."""
    provider_stop_sequence_key_name_map: Mapping[str, str] = {
        "anthropic": "stop_sequences",
        "amazon": "stopSequences",
        "ai21": "stop_sequences",
        "cohere": "stop_sequences",
    }
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that AWS credentials to and python package exists in environment."""
        # Skip creating new client if passed in constructor
        if values["client"] is not None:
            return values
        try:
            import boto3
            if values["credentials_profile_name"] is not None:
                session = boto3.Session(profile_name=values["credentials_profile_name"])
            else:
                # use default credentials
                session = boto3.Session()
            values["region_name"] = get_from_dict_or_env(
                values,
                "region_name",
                "AWS_DEFAULT_REGION",
                default=session.region_name,
            )
            client_params = {}
            if values["region_name"]:
                client_params["region_name"] = values["region_name"]
            if values["endpoint_url"]:
                client_params["endpoint_url"] = values["endpoint_url"]
            values["client"] = session.client("bedrock-runtime", **client_params)
        except ImportError:
            raise ModuleNotFoundError(
                "Could not import boto3 python package. "
raise ModuleNotFoundError(
                "Could not import boto3 python package. "
                "Please install it with `pip install boto3`."
            )
        except Exception as e:
            raise ValueError(
                "Could not load credentials to authenticate with AWS client. "
                "Please check that credentials in the specified "
                "profile name are valid."
            ) from e
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        _model_kwargs = self.model_kwargs or {}
        return {
            **{"model_kwargs": _model_kwargs},
        }
    def _get_provider(self) -> str:
        return self.model_id.split(".")[0]
    @property
    def _model_is_anthropic(self) -> bool:
        return self._get_provider() == "anthropic"
    def _prepare_input_and_invoke(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        _model_kwargs = self.model_kwargs or {}
        provider = self._get_provider()
        params = {**_model_kwargs, **kwargs}
        input_body = LLMInputOutputAdapter.prepare_input(provider, prompt, params)
        body = json.dumps(input_body)
        accept = "application/json"
        contentType = "application/json"
        try:
            response = self.client.invoke_model(
                body=body, modelId=self.model_id, accept=accept, contentType=contentType
            )
            text = LLMInputOutputAdapter.prepare_output(provider, response)
        except Exception as e:
except Exception as e:
            raise ValueError(f"Error raised by bedrock service: {e}")
        if stop is not None:
            text = enforce_stop_tokens(text, stop)
        return text
    def _prepare_input_and_invoke_stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        _model_kwargs = self.model_kwargs or {}
        provider = self._get_provider()
        if stop:
            if provider not in self.provider_stop_sequence_key_name_map:
                raise ValueError(
                    f"Stop sequence key name for {provider} is not supported."
                )
            # stop sequence from _generate() overrides
            # stop sequences in the class attribute
            _model_kwargs[self.provider_stop_sequence_key_name_map.get(provider)] = stop
        if provider == "cohere":
            _model_kwargs["stream"] = True
        params = {**_model_kwargs, **kwargs}
        input_body = LLMInputOutputAdapter.prepare_input(provider, prompt, params)
        body = json.dumps(input_body)
        try:
            response = self.client.invoke_model_with_response_stream(
                body=body,
                modelId=self.model_id,
                accept="application/json",
                contentType="application/json",
            )
        except Exception as e:
            raise ValueError(f"Error raised by bedrock service: {e}")
        for chunk in LLMInputOutputAdapter.prepare_output_stream(
            provider, response, stop
        ):
            yield chunk
            if run_manager is not None:
                run_manager.on_llm_new_token(chunk.text, chunk=chunk)
run_manager.on_llm_new_token(chunk.text, chunk=chunk)
[docs]class Bedrock(LLM, BedrockBase):
    """Bedrock models.
    To authenticate, the AWS client uses the following methods to
    automatically load credentials:
    https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    If a specific credential profile should be used, you must pass
    the name of the profile from the ~/.aws/credentials file that is to be used.
    Make sure the credentials / roles used have the required policies to
    access the Bedrock service.
    """
    """
    Example:
        .. code-block:: python
            from bedrock_langchain.bedrock_llm import BedrockLLM
            llm = BedrockLLM(
                credentials_profile_name="default",
                model_id="amazon.titan-text-express-v1",
                streaming=True
            )
    """
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return "amazon_bedrock"
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        """Return whether this model can be serialized by Langchain."""
        return True
[docs]    @classmethod
    def get_lc_namespace(cls) -> List[str]:
        """Get the namespace of the langchain object."""
        return ["langchain", "llms", "bedrock"]
    @property
    def lc_attributes(self) -> Dict[str, Any]:
        attributes: Dict[str, Any] = {}
        if self.region_name:
            attributes["region_name"] = self.region_name
        return attributes
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
"""Configuration for this pydantic object."""
        extra = Extra.forbid
    def _stream(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> Iterator[GenerationChunk]:
        """Call out to Bedrock service with streaming.
        Args:
            prompt (str): The prompt to pass into the model
            stop (Optional[List[str]], optional): Stop sequences. These will
                override any stop sequences in the `model_kwargs` attribute.
                Defaults to None.
            run_manager (Optional[CallbackManagerForLLMRun], optional): Callback
                run managers used to process the output. Defaults to None.
        Returns:
            Iterator[GenerationChunk]: Generator that yields the streamed responses.
        Yields:
            Iterator[GenerationChunk]: Responses from the model.
        """
        return self._prepare_input_and_invoke_stream(
            prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
        )
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Bedrock service model.
        Args:
            prompt: The prompt to pass into the model.
            stop: Optional list of stop words to use when generating.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = llm("Tell me a joke.")
        """
        if self.streaming:
            completion = ""
            for chunk in self._stream(
if self.streaming:
            completion = ""
            for chunk in self._stream(
                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
            ):
                completion += chunk.text
            return completion
        return self._prepare_input_and_invoke(prompt=prompt, stop=stop, **kwargs)
[docs]    def get_num_tokens(self, text: str) -> int:
        if self._model_is_anthropic:
            return get_num_tokens_anthropic(text)
        else:
            return super().get_num_tokens(text)
[docs]    def get_token_ids(self, text: str) -> List[int]:
        if self._model_is_anthropic:
            return get_token_ids_anthropic(text)
        else:
            return super().get_token_ids(text)
Source code for langchain_community.llms.aviary
import dataclasses
import os
from typing import Any, Dict, List, Mapping, Optional, Union, cast
import requests
from langchain_core.callbacks import CallbackManagerForLLMRun
from langchain_core.language_models.llms import LLM
from langchain_core.pydantic_v1 import Extra, root_validator
from langchain_core.utils import get_from_dict_or_env
from langchain_community.llms.utils import enforce_stop_tokens
TIMEOUT = 60
[docs]@dataclasses.dataclass
class AviaryBackend:
    """Aviary backend.
    Attributes:
        backend_url: The URL for the Aviary backend.
        bearer: The bearer token for the Aviary backend.
    """
    backend_url: str
    bearer: str
    def __post_init__(self) -> None:
        self.header = {"Authorization": self.bearer}
[docs]    @classmethod
    def from_env(cls) -> "AviaryBackend":
        aviary_url = os.getenv("AVIARY_URL")
        assert aviary_url, "AVIARY_URL must be set"
        aviary_token = os.getenv("AVIARY_TOKEN", "")
        bearer = f"Bearer {aviary_token}" if aviary_token else ""
        aviary_url += "/" if not aviary_url.endswith("/") else ""
        return cls(aviary_url, bearer)
[docs]def get_models() -> List[str]:
    """List available models"""
    backend = AviaryBackend.from_env()
    request_url = backend.backend_url + "-/routes"
    response = requests.get(request_url, headers=backend.header, timeout=TIMEOUT)
    try:
        result = response.json()
    except requests.JSONDecodeError as e:
        raise RuntimeError(
except requests.JSONDecodeError as e:
        raise RuntimeError(
            f"Error decoding JSON from {request_url}. Text response: {response.text}"
        ) from e
    result = sorted(
        [k.lstrip("/").replace("--", "/") for k in result.keys() if "--" in k]
    )
    return result
[docs]def get_completions(
    model: str,
    prompt: str,
    use_prompt_format: bool = True,
    version: str = "",
) -> Dict[str, Union[str, float, int]]:
    """Get completions from Aviary models."""
    backend = AviaryBackend.from_env()
    url = backend.backend_url + model.replace("/", "--") + "/" + version + "query"
    response = requests.post(
        url,
        headers=backend.header,
        json={"prompt": prompt, "use_prompt_format": use_prompt_format},
        timeout=TIMEOUT,
    )
    try:
        return response.json()
    except requests.JSONDecodeError as e:
        raise RuntimeError(
            f"Error decoding JSON from {url}. Text response: {response.text}"
        ) from e
[docs]class Aviary(LLM):
    """Aviary hosted models.
    Aviary is a backend for hosted models. You can
    find out more about aviary at
    http://github.com/ray-project/aviary
    To get a list of the models supported on an
    aviary, follow the instructions on the website to
    install the aviary CLI and then use:
    `aviary models`
    AVIARY_URL and AVIARY_TOKEN environment variables must be set.
    Attributes:
Attributes:
        model: The name of the model to use. Defaults to "amazon/LightGPT".
        aviary_url: The URL for the Aviary backend. Defaults to None.
        aviary_token: The bearer token for the Aviary backend. Defaults to None.
        use_prompt_format: If True, the prompt template for the model will be ignored.
            Defaults to True.
        version: API version to use for Aviary. Defaults to None.
    Example:
        .. code-block:: python
            from langchain_community.llms import Aviary
            os.environ["AVIARY_URL"] = "<URL>"
            os.environ["AVIARY_TOKEN"] = "<TOKEN>"
            light = Aviary(model='amazon/LightGPT')
            output = light('How do you make fried rice?')
    """
    model: str = "amazon/LightGPT"
    aviary_url: Optional[str] = None
    aviary_token: Optional[str] = None
    # If True the prompt template for the model will be ignored.
    use_prompt_format: bool = True
    # API version to use for Aviary
    version: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        aviary_url = get_from_dict_or_env(values, "aviary_url", "AVIARY_URL")
        aviary_token = get_from_dict_or_env(values, "aviary_token", "AVIARY_TOKEN")
        # Set env viarables for aviary sdk
# Set env viarables for aviary sdk
        os.environ["AVIARY_URL"] = aviary_url
        os.environ["AVIARY_TOKEN"] = aviary_token
        try:
            aviary_models = get_models()
        except requests.exceptions.RequestException as e:
            raise ValueError(e)
        model = values.get("model")
        if model and model not in aviary_models:
            raise ValueError(f"{aviary_url} does not support model {values['model']}.")
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model,
            "aviary_url": self.aviary_url,
        }
    @property
    def _llm_type(self) -> str:
        """Return type of llm."""
        return f"aviary-{self.model.replace('/', '-')}"
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any,
    ) -> str:
        """Call out to Aviary
        Args:
            prompt: The prompt to pass into the model.
        Returns:
            The string generated by the model.
        Example:
            .. code-block:: python
                response = aviary("Tell me a joke.")
        """
        kwargs = {"use_prompt_format": self.use_prompt_format}
        if self.version:
            kwargs["version"] = self.version
        output = get_completions(
            model=self.model,
            prompt=prompt,
            **kwargs,
        )
        text = cast(str, output["generated_text"])
Source code for langchain_community.llms.loading
"""Base interface for loading large language model APIs."""
import json
from pathlib import Path
from typing import Union
import yaml
from langchain_core.language_models.llms import BaseLLM
from langchain_community.llms import get_type_to_cls_dict
[docs]def load_llm_from_config(config: dict) -> BaseLLM:
    """Load LLM from Config Dict."""
    if "_type" not in config:
        raise ValueError("Must specify an LLM Type in config")
    config_type = config.pop("_type")
    type_to_cls_dict = get_type_to_cls_dict()
    if config_type not in type_to_cls_dict:
        raise ValueError(f"Loading {config_type} LLM not supported")
    llm_cls = type_to_cls_dict[config_type]()
    return llm_cls(**config)
[docs]def load_llm(file: Union[str, Path]) -> BaseLLM:
    """Load LLM from file."""
    # Convert file to Path object.
    if isinstance(file, str):
        file_path = Path(file)
    else:
        file_path = file
    # Load from either json or yaml.
    if file_path.suffix == ".json":
        with open(file_path) as f:
            config = json.load(f)
    elif file_path.suffix == ".yaml":
        with open(file_path, "r") as f:
            config = yaml.safe_load(f)
    else:
        raise ValueError("File type must be json or yaml")
    # Load the LLM from the config now.
    return load_llm_from_config(config)
Source code for langchain_community.llms.textgen
import json
import logging
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional
import requests
from langchain_core.callbacks import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain_core.language_models.llms import LLM
from langchain_core.outputs import GenerationChunk
from langchain_core.pydantic_v1 import Field
logger = logging.getLogger(__name__)
[docs]class TextGen(LLM):
    """text-generation-webui models.
    To use, you should have the text-generation-webui installed, a model loaded,
    and --api added as a command-line option.
    Suggested installation, use one-click installer for your OS:
    https://github.com/oobabooga/text-generation-webui#one-click-installers
    Parameters below taken from text-generation-webui api example:
    https://github.com/oobabooga/text-generation-webui/blob/main/api-examples/api-example.py
    Example:
        .. code-block:: python
            from langchain_community.llms import TextGen
            llm = TextGen(model_url="http://localhost:8500")
    """
    model_url: str
    """The full URL to the textgen webui including http[s]://host:port """
    preset: Optional[str] = None
    """The preset to use in the textgen webui """
    max_new_tokens: Optional[int] = 250
    """The maximum number of tokens to generate."""
    do_sample: bool = Field(True, alias="do_sample")
    """Do sample"""
    temperature: Optional[float] = 1.3
    """Primary factor to control randomness of outputs. 0 = deterministic
"""Primary factor to control randomness of outputs. 0 = deterministic
    (only the most likely token is used). Higher value = more randomness."""
    top_p: Optional[float] = 0.1
    """If not set to 1, select tokens with probabilities adding up to less than this
    number. Higher value = higher range of possible random results."""
    typical_p: Optional[float] = 1
    """If not set to 1, select only tokens that are at least this much more likely to
    appear than random tokens, given the prior text."""
    epsilon_cutoff: Optional[float] = 0  # In units of 1e-4
    """Epsilon cutoff"""
    eta_cutoff: Optional[float] = 0  # In units of 1e-4
    """ETA cutoff"""
    repetition_penalty: Optional[float] = 1.18
    """Exponential penalty factor for repeating prior tokens. 1 means no penalty,
    higher value = less repetition, lower value = more repetition."""
    top_k: Optional[float] = 40
    """Similar to top_p, but select instead only the top_k most likely tokens.
    Higher value = higher range of possible random results."""
    min_length: Optional[int] = 0
    """Minimum generation length in tokens."""
    no_repeat_ngram_size: Optional[int] = 0
    """If not set to 0, specifies the length of token sets that are completely blocked
    from repeating at all. Higher values = blocks larger phrases,
    lower values = blocks words or letters from repeating.
    Only 0 or high values are a good idea in most cases."""
    num_beams: Optional[int] = 1
    """Number of beams"""
    penalty_alpha: Optional[float] = 0
    """Penalty Alpha"""