Tony20100/langchain_full huggingface dataset:
# fixed window to allow 10 requests every 10 seconds:
                    limiter=FixedWindow(max_requests=10, window=10),
                )
                user_id = "foo"
                handler = UpstashRatelimitHandler(
                    identifier=user_id,
                    request_ratelimit=ratelimit
                )
                # Initialize a simple runnable to test
                chain = RunnableLambda(str)
                # pass handler as callback:
                output = chain.invoke(
                    "input",
                    config={
                        "callbacks": [handler]
                    }
                )
        """
        if not any([token_ratelimit, request_ratelimit]):
            raise ValueError(
                "You must pass at least one of input_token_ratelimit or"
                " request_ratelimit parameters for handler to work."
            )
        self.identifier = identifier
        self.token_ratelimit = token_ratelimit
        self.request_ratelimit = request_ratelimit
        self.include_output_tokens = include_output_tokens
[docs]    def on_chain_start(
        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
    ) -> Any:
        """
        Run when chain starts running.
        on_chain_start runs multiple times during a chain execution. To make
        sure that it's only called once, we keep a bool state `_checked`. If
        not `self._checked`, we call limit with `request_ratelimit` and raise
        `UpstashRatelimitError` if the identifier is rate limited.
        """
        if self.request_ratelimit and not self._checked:
            response = self.request_ratelimit.limit(self.identifier)
            if not response.allowed:
                raise UpstashRatelimitError(
if not response.allowed:
                raise UpstashRatelimitError(
                    "Request limit reached!", "request", response.limit, response.reset
                )
            self._checked = True
[docs]    def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """
        Run when LLM starts running
        """
        if self.token_ratelimit:
            remaining = self.token_ratelimit.get_remaining(self.identifier)
            if remaining <= 0:
                raise UpstashRatelimitError("Token limit reached!", "token")
[docs]    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """
        Run when LLM ends running
        If the `include_output_tokens` is set to True, number of tokens
        in LLM completion are counted for rate limiting
        """
        if self.token_ratelimit:
            try:
                llm_output = response.llm_output or {}
                token_usage = llm_output["token_usage"]
                token_count = (
                    token_usage["total_tokens"]
                    if self.include_output_tokens
                    else token_usage["prompt_tokens"]
                )
            except KeyError:
                raise ValueError(
                    "LLM response doesn't include"
                    " `token_usage: {total_tokens: int, prompt_tokens: int}`"
                    "  field. To use UpstashRatelimitHandler with token_ratelimit,"
                    " either use a model which returns token_usage (like "
                    " OpenAI models) or rate limit only with request_ratelimit."
                )
            # call limit to add the completion tokens to rate limit
            # but don't raise exception since we already generated
# but don't raise exception since we already generated
            # the tokens and would rather continue execution.
            self.token_ratelimit.limit(self.identifier, rate=token_count)
[docs]    def reset(self, identifier: Optional[str] = None) -> "UpstashRatelimitHandler":
        """
        Creates a new UpstashRatelimitHandler object with the same
        ratelimit configurations but with a new identifier if it's
        provided.
        Also resets the state of the handler.
        """
        return UpstashRatelimitHandler(
            identifier=identifier or self.identifier,
            token_ratelimit=self.token_ratelimit,
            request_ratelimit=self.request_ratelimit,
            include_output_tokens=self.include_output_tokens,
        )
Source code for langchain_community.callbacks.manager
from __future__ import annotations
import logging
from contextlib import contextmanager
from contextvars import ContextVar
from typing import (
    Generator,
    Optional,
)
from langchain_core.tracers.context import register_configure_hook
from langchain_community.callbacks.bedrock_anthropic_callback import (
    BedrockAnthropicTokenUsageCallbackHandler,
)
from langchain_community.callbacks.openai_info import OpenAICallbackHandler
from langchain_community.callbacks.tracers.comet import CometTracer
from langchain_community.callbacks.tracers.wandb import WandbTracer
logger = logging.getLogger(__name__)
openai_callback_var: ContextVar[Optional[OpenAICallbackHandler]] = ContextVar(
    "openai_callback", default=None
)
bedrock_anthropic_callback_var: (ContextVar)[
    Optional[BedrockAnthropicTokenUsageCallbackHandler]
] = ContextVar("bedrock_anthropic_callback", default=None)
wandb_tracing_callback_var: ContextVar[Optional[WandbTracer]] = ContextVar(
    "tracing_wandb_callback", default=None
)
comet_tracing_callback_var: ContextVar[Optional[CometTracer]] = ContextVar(
    "tracing_comet_callback", default=None
)
register_configure_hook(openai_callback_var, True)
register_configure_hook(bedrock_anthropic_callback_var, True)
register_configure_hook(
    wandb_tracing_callback_var, True, WandbTracer, "LANGCHAIN_WANDB_TRACING"
)
register_configure_hook(
    comet_tracing_callback_var, True, CometTracer, "LANGCHAIN_COMET_TRACING"
)
[docs]@contextmanager
def get_openai_callback() -> Generator[OpenAICallbackHandler, None, None]:
def get_openai_callback() -> Generator[OpenAICallbackHandler, None, None]:
    """Get the OpenAI callback handler in a context manager.
    which conveniently exposes token and cost information.
    Returns:
        OpenAICallbackHandler: The OpenAI callback handler.
    Example:
        >>> with get_openai_callback() as cb:
        ...     # Use the OpenAI callback handler
    """
    cb = OpenAICallbackHandler()
    openai_callback_var.set(cb)
    yield cb
    openai_callback_var.set(None)
[docs]@contextmanager
def get_bedrock_anthropic_callback() -> (
    Generator[BedrockAnthropicTokenUsageCallbackHandler, None, None]
):
    """Get the Bedrock anthropic callback handler in a context manager.
    which conveniently exposes token and cost information.
    Returns:
        BedrockAnthropicTokenUsageCallbackHandler:
            The Bedrock anthropic callback handler.
    Example:
        >>> with get_bedrock_anthropic_callback() as cb:
        ...     # Use the Bedrock anthropic callback handler
    """
    cb = BedrockAnthropicTokenUsageCallbackHandler()
    bedrock_anthropic_callback_var.set(cb)
    yield cb
    bedrock_anthropic_callback_var.set(None)
[docs]@contextmanager
def wandb_tracing_enabled(
    session_name: str = "default",
) -> Generator[None, None, None]:
    """Get the WandbTracer in a context manager.
    Args:
        session_name (str, optional): The name of the session.
            Defaults to "default".
    Returns:
        None
    Example:
        >>> with wandb_tracing_enabled() as session:
Source code for langchain_community.callbacks.comet_ml_callback
import tempfile
from copy import deepcopy
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Sequence
from langchain_core.agents import AgentAction, AgentFinish
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.outputs import Generation, LLMResult
from langchain_core.utils import guard_import
import langchain_community
from langchain_community.callbacks.utils import (
    BaseMetadataCallbackHandler,
    flatten_dict,
    import_pandas,
    import_spacy,
    import_textstat,
)
LANGCHAIN_MODEL_NAME = "langchain-model"
[docs]def import_comet_ml() -> Any:
    """Import comet_ml and raise an error if it is not installed."""
    return guard_import("comet_ml")
def _get_experiment(
    workspace: Optional[str] = None, project_name: Optional[str] = None
) -> Any:
    comet_ml = import_comet_ml()
    experiment = comet_ml.Experiment(
        workspace=workspace,
        project_name=project_name,
    )
    return experiment
def _fetch_text_complexity_metrics(text: str) -> dict:
    textstat = import_textstat()
    text_complexity_metrics = {
        "flesch_reading_ease": textstat.flesch_reading_ease(text),
        "flesch_kincaid_grade": textstat.flesch_kincaid_grade(text),
        "smog_index": textstat.smog_index(text),
        "coleman_liau_index": textstat.coleman_liau_index(text),
        "automated_readability_index": textstat.automated_readability_index(text),
        "dale_chall_readability_score": textstat.dale_chall_readability_score(text),
"difficult_words": textstat.difficult_words(text),
        "linsear_write_formula": textstat.linsear_write_formula(text),
        "gunning_fog": textstat.gunning_fog(text),
        "text_standard": textstat.text_standard(text),
        "fernandez_huerta": textstat.fernandez_huerta(text),
        "szigriszt_pazos": textstat.szigriszt_pazos(text),
        "gutierrez_polini": textstat.gutierrez_polini(text),
        "crawford": textstat.crawford(text),
        "gulpease_index": textstat.gulpease_index(text),
        "osman": textstat.osman(text),
    }
    return text_complexity_metrics
def _summarize_metrics_for_generated_outputs(metrics: Sequence) -> dict:
    pd = import_pandas()
    metrics_df = pd.DataFrame(metrics)
    metrics_summary = metrics_df.describe()
    return metrics_summary.to_dict()
[docs]class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
    """Callback Handler that logs to Comet.
    Parameters:
        job_type (str): The type of comet_ml task such as "inference",
            "testing" or "qc"
        project_name (str): The comet_ml project name
        tags (list): Tags to add to the task
        task_name (str): Name of the comet_ml task
        visualize (bool): Whether to visualize the run.
        complexity_metrics (bool): Whether to log complexity metrics
        stream_logs (bool): Whether to stream callback actions to Comet
    This handler will utilize the associated callback method and formats
    the input of each callback function with metadata regarding the state of LLM run,
the input of each callback function with metadata regarding the state of LLM run,
    and adds the response to the list of records for both the {method}_records and
    action. It then logs the response to Comet.
    """
[docs]    def __init__(
        self,
        task_type: Optional[str] = "inference",
        workspace: Optional[str] = None,
        project_name: Optional[str] = None,
        tags: Optional[Sequence] = None,
        name: Optional[str] = None,
        visualizations: Optional[List[str]] = None,
        complexity_metrics: bool = False,
        custom_metrics: Optional[Callable] = None,
        stream_logs: bool = True,
    ) -> None:
        """Initialize callback handler."""
        self.comet_ml = import_comet_ml()
        super().__init__()
        self.task_type = task_type
        self.workspace = workspace
        self.project_name = project_name
        self.tags = tags
        self.visualizations = visualizations
        self.complexity_metrics = complexity_metrics
        self.custom_metrics = custom_metrics
        self.stream_logs = stream_logs
        self.temp_dir = tempfile.TemporaryDirectory()
        self.experiment = _get_experiment(workspace, project_name)
        self.experiment.log_other("Created from", "langchain")
        if tags:
            self.experiment.add_tags(tags)
        self.name = name
        if self.name:
            self.experiment.set_name(self.name)
        warning = (
            "The comet_ml callback is currently in beta and is subject to change "
            "based on updates to `langchain`. Please report any issues to "
            "https://github.com/comet-ml/issue-tracking/issues with the tag "
            "`langchain`."
        )
"`langchain`."
        )
        self.comet_ml.LOGGER.warning(warning)
        self.callback_columns: list = []
        self.action_records: list = []
        self.complexity_metrics = complexity_metrics
        if self.visualizations:
            spacy = import_spacy()
            self.nlp = spacy.load("en_core_web_sm")
        else:
            self.nlp = None
    def _init_resp(self) -> Dict:
        return {k: None for k in self.callback_columns}
[docs]    def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """Run when LLM starts."""
        self.step += 1
        self.llm_starts += 1
        self.starts += 1
        metadata = self._init_resp()
        metadata.update({"action": "on_llm_start"})
        metadata.update(flatten_dict(serialized))
        metadata.update(self.get_custom_callback_meta())
        for prompt in prompts:
            prompt_resp = deepcopy(metadata)
            prompt_resp["prompts"] = prompt
            self.on_llm_start_records.append(prompt_resp)
            self.action_records.append(prompt_resp)
            if self.stream_logs:
                self._log_stream(prompt, metadata, self.step)
[docs]    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        """Run when LLM generates a new token."""
        self.step += 1
        self.llm_streams += 1
        resp = self._init_resp()
        resp.update({"action": "on_llm_new_token", "token": token})
        resp.update(self.get_custom_callback_meta())
        self.action_records.append(resp)
resp.update(self.get_custom_callback_meta())
        self.action_records.append(resp)
[docs]    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """Run when LLM ends running."""
        self.step += 1
        self.llm_ends += 1
        self.ends += 1
        metadata = self._init_resp()
        metadata.update({"action": "on_llm_end"})
        metadata.update(flatten_dict(response.llm_output or {}))
        metadata.update(self.get_custom_callback_meta())
        output_complexity_metrics = []
        output_custom_metrics = []
        for prompt_idx, generations in enumerate(response.generations):
            for gen_idx, generation in enumerate(generations):
                text = generation.text
                generation_resp = deepcopy(metadata)
                generation_resp.update(flatten_dict(generation.dict()))
                complexity_metrics = self._get_complexity_metrics(text)
                if complexity_metrics:
                    output_complexity_metrics.append(complexity_metrics)
                    generation_resp.update(complexity_metrics)
                custom_metrics = self._get_custom_metrics(
                    generation, prompt_idx, gen_idx
                )
                if custom_metrics:
                    output_custom_metrics.append(custom_metrics)
                    generation_resp.update(custom_metrics)
                if self.stream_logs:
                    self._log_stream(text, metadata, self.step)
                self.action_records.append(generation_resp)
                self.on_llm_end_records.append(generation_resp)
        self._log_text_metrics(output_complexity_metrics, step=self.step)
        self._log_text_metrics(output_custom_metrics, step=self.step)
[docs]    def on_llm_error(self, error: BaseException, **kwargs: Any) -> None:
        """Run when LLM errors."""
        self.step += 1
        self.errors += 1
self.step += 1
        self.errors += 1
[docs]    def on_chain_start(
        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
    ) -> None:
        """Run when chain starts running."""
        self.step += 1
        self.chain_starts += 1
        self.starts += 1
        resp = self._init_resp()
        resp.update({"action": "on_chain_start"})
        resp.update(flatten_dict(serialized))
        resp.update(self.get_custom_callback_meta())
        for chain_input_key, chain_input_val in inputs.items():
            if isinstance(chain_input_val, str):
                input_resp = deepcopy(resp)
                if self.stream_logs:
                    self._log_stream(chain_input_val, resp, self.step)
                input_resp.update({chain_input_key: chain_input_val})
                self.action_records.append(input_resp)
            else:
                self.comet_ml.LOGGER.warning(
                    f"Unexpected data format provided! "
                    f"Input Value for {chain_input_key} will not be logged"
                )
[docs]    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
        """Run when chain ends running."""
        self.step += 1
        self.chain_ends += 1
        self.ends += 1
        resp = self._init_resp()
        resp.update({"action": "on_chain_end"})
        resp.update(self.get_custom_callback_meta())
        for chain_output_key, chain_output_val in outputs.items():
            if isinstance(chain_output_val, str):
                output_resp = deepcopy(resp)
                if self.stream_logs:
                    self._log_stream(chain_output_val, resp, self.step)
self._log_stream(chain_output_val, resp, self.step)
                output_resp.update({chain_output_key: chain_output_val})
                self.action_records.append(output_resp)
            else:
                self.comet_ml.LOGGER.warning(
                    f"Unexpected data format provided! "
                    f"Output Value for {chain_output_key} will not be logged"
                )
[docs]    def on_chain_error(self, error: BaseException, **kwargs: Any) -> None:
        """Run when chain errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_tool_start(
        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
    ) -> None:
        """Run when tool starts running."""
        self.step += 1
        self.tool_starts += 1
        self.starts += 1
        resp = self._init_resp()
        resp.update({"action": "on_tool_start"})
        resp.update(flatten_dict(serialized))
        resp.update(self.get_custom_callback_meta())
        if self.stream_logs:
            self._log_stream(input_str, resp, self.step)
        resp.update({"input_str": input_str})
        self.action_records.append(resp)
[docs]    def on_tool_end(self, output: Any, **kwargs: Any) -> None:
        """Run when tool ends running."""
        output = str(output)
        self.step += 1
        self.tool_ends += 1
        self.ends += 1
        resp = self._init_resp()
        resp.update({"action": "on_tool_end"})
        resp.update(self.get_custom_callback_meta())
        if self.stream_logs:
            self._log_stream(output, resp, self.step)
        resp.update({"output": output})
resp.update({"output": output})
        self.action_records.append(resp)
[docs]    def on_tool_error(self, error: BaseException, **kwargs: Any) -> None:
        """Run when tool errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_text(self, text: str, **kwargs: Any) -> None:
        """
        Run when agent is ending.
        """
        self.step += 1
        self.text_ctr += 1
        resp = self._init_resp()
        resp.update({"action": "on_text"})
        resp.update(self.get_custom_callback_meta())
        if self.stream_logs:
            self._log_stream(text, resp, self.step)
        resp.update({"text": text})
        self.action_records.append(resp)
[docs]    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None:
        """Run when agent ends running."""
        self.step += 1
        self.agent_ends += 1
        self.ends += 1
        resp = self._init_resp()
        output = finish.return_values["output"]
        log = finish.log
        resp.update({"action": "on_agent_finish", "log": log})
        resp.update(self.get_custom_callback_meta())
        if self.stream_logs:
            self._log_stream(output, resp, self.step)
        resp.update({"output": output})
        self.action_records.append(resp)
[docs]    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
        """Run on agent action."""
        self.step += 1
        self.tool_starts += 1
        self.starts += 1
        tool = action.tool
        tool_input = str(action.tool_input)
tool = action.tool
        tool_input = str(action.tool_input)
        log = action.log
        resp = self._init_resp()
        resp.update({"action": "on_agent_action", "log": log, "tool": tool})
        resp.update(self.get_custom_callback_meta())
        if self.stream_logs:
            self._log_stream(tool_input, resp, self.step)
        resp.update({"tool_input": tool_input})
        self.action_records.append(resp)
    def _get_complexity_metrics(self, text: str) -> dict:
        """Compute text complexity metrics using textstat.
        Parameters:
            text (str): The text to analyze.
        Returns:
            (dict): A dictionary containing the complexity metrics.
        """
        resp = {}
        if self.complexity_metrics:
            text_complexity_metrics = _fetch_text_complexity_metrics(text)
            resp.update(text_complexity_metrics)
        return resp
    def _get_custom_metrics(
        self, generation: Generation, prompt_idx: int, gen_idx: int
    ) -> dict:
        """Compute Custom Metrics for an LLM Generated Output
        Args:
            generation (LLMResult): Output generation from an LLM
            prompt_idx (int): List index of the input prompt
            gen_idx (int): List index of the generated output
        Returns:
            dict: A dictionary containing the custom metrics.
        """
        resp = {}
        if self.custom_metrics:
            custom_metrics = self.custom_metrics(generation, prompt_idx, gen_idx)
            resp.update(custom_metrics)
        return resp
[docs]    def flush_tracker(
        self,
        langchain_asset: Any = None,
        task_type: Optional[str] = "inference",
        workspace: Optional[str] = None,
workspace: Optional[str] = None,
        project_name: Optional[str] = "comet-langchain-demo",
        tags: Optional[Sequence] = None,
        name: Optional[str] = None,
        visualizations: Optional[List[str]] = None,
        complexity_metrics: bool = False,
        custom_metrics: Optional[Callable] = None,
        finish: bool = False,
        reset: bool = False,
    ) -> None:
        """Flush the tracker and setup the session.
        Everything after this will be a new table.
        Args:
            name: Name of the performed session so far so it is identifiable
            langchain_asset: The langchain asset to save.
            finish: Whether to finish the run.
            Returns:
                None
        """
        self._log_session(langchain_asset)
        if langchain_asset:
            try:
                self._log_model(langchain_asset)
            except Exception:
                self.comet_ml.LOGGER.error(
                    "Failed to export agent or LLM to Comet",
                    exc_info=True,
                    extra={"show_traceback": True},
                )
        if finish:
            self.experiment.end()
        if reset:
            self._reset(
                task_type,
                workspace,
                project_name,
                tags,
                name,
                visualizations,
                complexity_metrics,
                custom_metrics,
            )
    def _log_stream(self, prompt: str, metadata: dict, step: int) -> None:
        self.experiment.log_text(prompt, metadata=metadata, step=step)
    def _log_model(self, langchain_asset: Any) -> None:
        model_parameters = self._get_llm_parameters(langchain_asset)