Tony20100/langchain_full huggingface dataset:
) -> None:
        """Print out the prompts."""
        pass
[docs]    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        """Print out the token."""
        pass
[docs]    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """Collect token usage."""
        if response.llm_output is None:
            return None
        if "usage" not in response.llm_output:
            with self._lock:
                self.successful_requests += 1
            return None
        # compute tokens and cost for this request
        token_usage = response.llm_output["usage"]
        completion_tokens = token_usage.get("completion_tokens", 0)
        prompt_tokens = token_usage.get("prompt_tokens", 0)
        total_tokens = token_usage.get("total_tokens", 0)
        model_id = response.llm_output.get("model_id", None)
        total_cost = _get_anthropic_claude_token_cost(
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
            model_id=model_id,
        )
        # update shared state behind lock
        with self._lock:
            self.total_cost += total_cost
            self.total_tokens += total_tokens
            self.prompt_tokens += prompt_tokens
            self.completion_tokens += completion_tokens
            self.successful_requests += 1
    def __copy__(self) -> "BedrockAnthropicTokenUsageCallbackHandler":
        """Return a copy of the callback handler."""
        return self
    def __deepcopy__(self, memo: Any) -> "BedrockAnthropicTokenUsageCallbackHandler":
        """Return a deep copy of the callback handler."""
        return self
for key in keys
            if message.additional_kwargs.get(key) is not None
        }
    )
    return parsed
def _parse_lc_messages(messages: Union[List[BaseMessage], Any]) -> List[Dict[str, Any]]:
    return [_parse_lc_message(message) for message in messages]
[docs]class LLMonitorCallbackHandler(BaseCallbackHandler):
    """Callback Handler for LLMonitor`.
    #### Parameters:
        - `app_id`: The app id of the app you want to report to. Defaults to
        `None`, which means that `LLMONITOR_APP_ID` will be used.
        - `api_url`: The url of the LLMonitor API. Defaults to `None`,
        which means that either `LLMONITOR_API_URL` environment variable
        or `https://app.llmonitor.com` will be used.
    #### Raises:
        - `ValueError`: if `app_id` is not provided either as an
        argument or as an environment variable.
        - `ConnectionError`: if the connection to the API fails.
    #### Example:
    ```python
    from langchain_community.llms import OpenAI
    from langchain_community.callbacks import LLMonitorCallbackHandler
    llmonitor_callback = LLMonitorCallbackHandler()
    llm = OpenAI(callbacks=[llmonitor_callback],
                 metadata={"userId": "user-123"})
    llm.invoke("Hello, how are you?")
    ```
    """
    __api_url: str
    __app_id: str
    __verbose: bool
    __llmonitor_version: str
    __has_valid_config: bool
[docs]    def __init__(
        self,
        app_id: Union[str, None] = None,
        api_url: Union[str, None] = None,
output=parsed_output,
                token_usage={
                    "prompt": token_usage.get("prompt_tokens"),
                    "completion": token_usage.get("completion_tokens"),
                },
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_llm_end: {e}")
[docs]    def on_tool_start(
        self,
        serialized: Dict[str, Any],
        input_str: str,
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
        tags: Union[List[str], None] = None,
        metadata: Union[Dict[str, Any], None] = None,
        **kwargs: Any,
    ) -> None:
        if self.__has_valid_config is False:
            return
        try:
            user_id = _get_user_id(metadata)
            user_props = _get_user_props(metadata)
            name = serialized.get("name")
            self.__track_event(
                "tool",
                "start",
                user_id=user_id,
                run_id=str(run_id),
                parent_run_id=str(parent_run_id) if parent_run_id else None,
                name=name,
                input=input_str,
                tags=tags,
                metadata=metadata,
                user_props=user_props,
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_tool_start: {e}")
[docs]    def on_tool_end(
        self,
        output: Any,
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
parent_run_id: Union[UUID, None] = None,
        tags: Union[List[str], None] = None,
        **kwargs: Any,
    ) -> None:
        output = str(output)
        if self.__has_valid_config is False:
            return
        try:
            self.__track_event(
                "tool",
                "end",
                run_id=str(run_id),
                parent_run_id=str(parent_run_id) if parent_run_id else None,
                output=output,
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_tool_end: {e}")
[docs]    def on_chain_start(
        self,
        serialized: Dict[str, Any],
        inputs: Dict[str, Any],
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
        tags: Union[List[str], None] = None,
        metadata: Union[Dict[str, Any], None] = None,
        **kwargs: Any,
    ) -> Any:
        if self.__has_valid_config is False:
            return
        try:
            name = serialized.get("id", [None, None, None, None])[3]
            type = "chain"
            metadata = metadata or {}
            agentName = metadata.get("agent_name")
            if agentName is None:
                agentName = metadata.get("agentName")
            if name == "AgentExecutor" or name == "PlanAndExecute":
                type = "agent"
            if agentName is not None:
                type = "agent"
                name = agentName
            if parent_run_id is not None:
                type = "chain"
if parent_run_id is not None:
                type = "chain"
            user_id = _get_user_id(metadata)
            user_props = _get_user_props(metadata)
            input = _parse_input(inputs)
            self.__track_event(
                type,
                "start",
                user_id=user_id,
                run_id=str(run_id),
                parent_run_id=str(parent_run_id) if parent_run_id else None,
                name=name,
                input=input,
                tags=tags,
                metadata=metadata,
                user_props=user_props,
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_chain_start: {e}")
[docs]    def on_chain_end(
        self,
        outputs: Dict[str, Any],
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
        **kwargs: Any,
    ) -> Any:
        if self.__has_valid_config is False:
            return
        try:
            output = _parse_output(outputs)
            self.__track_event(
                "chain",
                "end",
                run_id=str(run_id),
                parent_run_id=str(parent_run_id) if parent_run_id else None,
                output=output,
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_chain_end: {e}")
[docs]    def on_agent_action(
        self,
        action: AgentAction,
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
        **kwargs: Any,
    ) -> Any:
**kwargs: Any,
    ) -> Any:
        if self.__has_valid_config is False:
            return
        try:
            name = action.tool
            input = _parse_input(action.tool_input)
            self.__track_event(
                "tool",
                "start",
                run_id=str(run_id),
                parent_run_id=str(parent_run_id) if parent_run_id else None,
                name=name,
                input=input,
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_agent_action: {e}")
[docs]    def on_agent_finish(
        self,
        finish: AgentFinish,
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
        **kwargs: Any,
    ) -> Any:
        if self.__has_valid_config is False:
            return
        try:
            output = _parse_output(finish.return_values)
            self.__track_event(
                "agent",
                "end",
                run_id=str(run_id),
                parent_run_id=str(parent_run_id) if parent_run_id else None,
                output=output,
                app_id=self.__app_id,
            )
        except Exception as e:
            logger.error(f"[LLMonitor] An error occurred in on_agent_finish: {e}")
[docs]    def on_chain_error(
        self,
        error: BaseException,
        *,
        run_id: UUID,
        parent_run_id: Union[UUID, None] = None,
        **kwargs: Any,
    ) -> Any:
        if self.__has_valid_config is False:
            return
        try:
            self.__track_event(
Source code for langchain_community.callbacks.utils
import hashlib
from pathlib import Path
from typing import Any, Dict, Iterable, Tuple, Union
from langchain_core.utils import guard_import
[docs]def import_spacy() -> Any:
    """Import the spacy python package and raise an error if it is not installed."""
    return guard_import("spacy")
[docs]def import_pandas() -> Any:
    """Import the pandas python package and raise an error if it is not installed."""
    return guard_import("pandas")
[docs]def import_textstat() -> Any:
    """Import the textstat python package and raise an error if it is not installed."""
    return guard_import("textstat")
def _flatten_dict(
    nested_dict: Dict[str, Any], parent_key: str = "", sep: str = "_"
) -> Iterable[Tuple[str, Any]]:
    """
    Generator that yields flattened items from a nested dictionary for a flat dict.
    Parameters:
        nested_dict (dict): The nested dictionary to flatten.
        parent_key (str): The prefix to prepend to the keys of the flattened dict.
        sep (str): The separator to use between the parent key and the key of the
            flattened dictionary.
    Yields:
        (str, any): A key-value pair from the flattened dictionary.
    """
    for key, value in nested_dict.items():
        new_key = parent_key + sep + key if parent_key else key
        if isinstance(value, dict):
            yield from _flatten_dict(value, new_key, sep)
        else:
            yield new_key, value
[docs]def flatten_dict(
    nested_dict: Dict[str, Any], parent_key: str = "", sep: str = "_"
) -> Dict[str, Any]:
) -> Dict[str, Any]:
    """Flatten a nested dictionary into a flat dictionary.
    Parameters:
        nested_dict (dict): The nested dictionary to flatten.
        parent_key (str): The prefix to prepend to the keys of the flattened dict.
        sep (str): The separator to use between the parent key and the key of the
            flattened dictionary.
    Returns:
        (dict): A flat dictionary.
    """
    flat_dict = {k: v for k, v in _flatten_dict(nested_dict, parent_key, sep)}
    return flat_dict
[docs]def hash_string(s: str) -> str:
    """Hash a string using sha1.
    Parameters:
        s (str): The string to hash.
    Returns:
        (str): The hashed string.
    """
    return hashlib.sha1(s.encode("utf-8")).hexdigest()
[docs]def load_json(json_path: Union[str, Path]) -> str:
    """Load json file to a string.
    Parameters:
        json_path (str): The path to the json file.
    Returns:
        (str): The string representation of the json file.
    """
    with open(json_path, "r") as f:
        data = f.read()
    return data
[docs]class BaseMetadataCallbackHandler:
    """Handle the metadata and associated function states for callbacks.
    Attributes:
        step (int): The current step.
        starts (int): The number of times the start method has been called.
        ends (int): The number of times the end method has been called.
        errors (int): The number of times the error method has been called.
        text_ctr (int): The number of times the text method has been called.
text_ctr (int): The number of times the text method has been called.
        ignore_llm_ (bool): Whether to ignore llm callbacks.
        ignore_chain_ (bool): Whether to ignore chain callbacks.
        ignore_agent_ (bool): Whether to ignore agent callbacks.
        ignore_retriever_ (bool): Whether to ignore retriever callbacks.
        always_verbose_ (bool): Whether to always be verbose.
        chain_starts (int): The number of times the chain start method has been called.
        chain_ends (int): The number of times the chain end method has been called.
        llm_starts (int): The number of times the llm start method has been called.
        llm_ends (int): The number of times the llm end method has been called.
        llm_streams (int): The number of times the text method has been called.
        tool_starts (int): The number of times the tool start method has been called.
        tool_ends (int): The number of times the tool end method has been called.
        agent_ends (int): The number of times the agent end method has been called.
        on_llm_start_records (list): A list of records of the on_llm_start method.
        on_llm_token_records (list): A list of records of the on_llm_token method.
        on_llm_end_records (list): A list of records of the on_llm_end method.
        on_chain_start_records (list): A list of records of the on_chain_start method.
        on_chain_end_records (list): A list of records of the on_chain_end method.
        on_tool_start_records (list): A list of records of the on_tool_start method.
        on_tool_end_records (list): A list of records of the on_tool_end method.
on_tool_end_records (list): A list of records of the on_tool_end method.
        on_agent_finish_records (list): A list of records of the on_agent_end method.
    """
[docs]    def __init__(self) -> None:
        self.step = 0
        self.starts = 0
        self.ends = 0
        self.errors = 0
        self.text_ctr = 0
        self.ignore_llm_ = False
        self.ignore_chain_ = False
        self.ignore_agent_ = False
        self.ignore_retriever_ = False
        self.always_verbose_ = False
        self.chain_starts = 0
        self.chain_ends = 0
        self.llm_starts = 0
        self.llm_ends = 0
        self.llm_streams = 0
        self.tool_starts = 0
        self.tool_ends = 0
        self.agent_ends = 0
        self.on_llm_start_records: list = []
        self.on_llm_token_records: list = []
        self.on_llm_end_records: list = []
        self.on_chain_start_records: list = []
        self.on_chain_end_records: list = []
        self.on_tool_start_records: list = []
        self.on_tool_end_records: list = []
        self.on_text_records: list = []
        self.on_agent_finish_records: list = []
        self.on_agent_action_records: list = []
    @property
    def always_verbose(self) -> bool:
        """Whether to call verbose callbacks even if verbose is False."""
        return self.always_verbose_
    @property
    def ignore_llm(self) -> bool:
        """Whether to ignore LLM callbacks."""
        return self.ignore_llm_
    @property
    def ignore_chain(self) -> bool:
@property
    def ignore_chain(self) -> bool:
        """Whether to ignore chain callbacks."""
        return self.ignore_chain_
    @property
    def ignore_agent(self) -> bool:
        """Whether to ignore agent callbacks."""
        return self.ignore_agent_
[docs]    def get_custom_callback_meta(self) -> Dict[str, Any]:
        return {
            "step": self.step,
            "starts": self.starts,
            "ends": self.ends,
            "errors": self.errors,
            "text_ctr": self.text_ctr,
            "chain_starts": self.chain_starts,
            "chain_ends": self.chain_ends,
            "llm_starts": self.llm_starts,
            "llm_ends": self.llm_ends,
            "llm_streams": self.llm_streams,
            "tool_starts": self.tool_starts,
            "tool_ends": self.tool_ends,
            "agent_ends": self.agent_ends,
        }
[docs]    def reset_callback_meta(self) -> None:
        """Reset the callback metadata."""
        self.step = 0
        self.starts = 0
        self.ends = 0
        self.errors = 0
        self.text_ctr = 0
        self.ignore_llm_ = False
        self.ignore_chain_ = False
        self.ignore_agent_ = False
        self.always_verbose_ = False
        self.chain_starts = 0
        self.chain_ends = 0
        self.llm_starts = 0
        self.llm_ends = 0
        self.llm_streams = 0
        self.tool_starts = 0
        self.tool_ends = 0
        self.agent_ends = 0
        self.on_llm_start_records = []
        self.on_llm_token_records = []
self.on_llm_start_records = []
        self.on_llm_token_records = []
        self.on_llm_end_records = []
        self.on_chain_start_records = []
        self.on_chain_end_records = []
        self.on_tool_start_records = []
        self.on_tool_end_records = []
        self.on_text_records = []
        self.on_agent_finish_records = []
        self.on_agent_action_records = []
        return None
resp: Dict[str, Any] = {}
        resp.update({"action": "on_tool_start", "input_str": input_str})
        resp.update(flatten_dict(serialized))
        resp.update(self.metrics)
        self.jsonf(resp, self.temp_dir, f"tool_start_{tool_starts}")
[docs]    def on_tool_end(self, output: Any, **kwargs: Any) -> None:
        """Run when tool ends running."""
        output = str(output)
        self.metrics["step"] += 1
        self.metrics["tool_ends"] += 1
        self.metrics["ends"] += 1
        tool_ends = self.metrics["tool_ends"]
        resp: Dict[str, Any] = {}
        resp.update({"action": "on_tool_end", "output": output})
        resp.update(self.metrics)
        self.jsonf(resp, self.temp_dir, f"tool_end_{tool_ends}")
[docs]    def on_tool_error(self, error: BaseException, **kwargs: Any) -> None:
        """Run when tool errors."""
        self.metrics["step"] += 1
        self.metrics["errors"] += 1
[docs]    def on_text(self, text: str, **kwargs: Any) -> None:
        """
        Run when agent is ending.
        """
        self.metrics["step"] += 1
        self.metrics["text_ctr"] += 1
        text_ctr = self.metrics["text_ctr"]
        resp: Dict[str, Any] = {}
        resp.update({"action": "on_text", "text": text})
        resp.update(self.metrics)
        self.jsonf(resp, self.temp_dir, f"on_text_{text_ctr}")
[docs]    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None:
Source code for langchain_community.callbacks.upstash_ratelimit_callback
"""Ratelimiting Handler to limit requests or tokens"""
import logging
from typing import Any, Dict, List, Literal, Optional
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.outputs import LLMResult
logger = logging.getLogger(__name__)
try:
    from upstash_ratelimit import Ratelimit
except ImportError:
    Ratelimit = None
[docs]class UpstashRatelimitError(Exception):
    """
    Upstash Ratelimit Error
    Raised when the rate limit is reached in `UpstashRatelimitHandler`
    """
    def __init__(
        self,
        message: str,
        type: Literal["token", "request"],
        limit: Optional[int] = None,
        reset: Optional[float] = None,
    ):
        """
        Args:
            message (str): error message
            type (str): The kind of the limit which was reached. One of
                "token" or "request"
            limit (Optional[int]): The limit which was reached. Passed when type
                is request
            reset (Optional[int]): unix timestamp in milliseconds when the limits
                are reset. Passed when type is request
        """
        # Call the base class constructor with the parameters it needs
        super().__init__(message)
        self.type = type
        self.limit = limit
        self.reset = reset
[docs]class UpstashRatelimitHandler(BaseCallbackHandler):
    """
    Callback to handle rate limiting based on the number of requests
    or the number of tokens in the input.
    It uses Upstash Ratelimit to track the ratelimit which utilizes
    Upstash Redis to track the state.
Upstash Redis to track the state.
    Should not be passed to the chain when initialising the chain.
    This is because the handler has a state which should be fresh
    every time invoke is called. Instead, initialise and pass a handler
    every time you invoke.
    """
    raise_error = True
    _checked = False
[docs]    def __init__(
        self,
        identifier: str,
        *,
        token_ratelimit: Optional[Ratelimit] = None,
        request_ratelimit: Optional[Ratelimit] = None,
        include_output_tokens: bool = False,
    ):
        """
        Creates UpstashRatelimitHandler. Must be passed an identifier to
        ratelimit like a user id or an ip address.
        Additionally, it must be passed at least one of token_ratelimit
        or request_ratelimit parameters.
        Args:
            identifier Union[int, str]: the identifier
            token_ratelimit Optional[Ratelimit]: Ratelimit to limit the
                number of tokens. Only works with OpenAI models since only
                these models provide the number of tokens as information
                in their output.
            request_ratelimit Optional[Ratelimit]: Ratelimit to limit the
                number of requests
            include_output_tokens bool: Whether to count output tokens when
                rate limiting based on number of tokens. Only used when
                `token_ratelimit` is passed. False by default.
        Example:
            .. code-block:: python
                from upstash_redis import Redis
                from upstash_ratelimit import Ratelimit, FixedWindow
                redis = Redis.from_env()
                ratelimit = Ratelimit(
                    redis=redis,
                    # fixed window to allow 10 requests every 10 seconds: