viruPVS/langchain-docs huggingface dataset:
visualizations,
                complexity_metrics,
                custom_metrics,
            )
    def _log_stream(self, prompt: str, metadata: dict, step: int) -> None:
        self.experiment.log_text(prompt, metadata=metadata, step=step)
    def _log_model(self, langchain_asset: Any) -> None:
        model_parameters = self._get_llm_parameters(langchain_asset)
        self.experiment.log_parameters(model_parameters, prefix="model")
        langchain_asset_path = Path(self.temp_dir.name, "model.json")
        model_name = self.name if self.name else LANGCHAIN_MODEL_NAME
        try:
            if hasattr(langchain_asset, "save"):
                langchain_asset.save(langchain_asset_path)
                self.experiment.log_model(model_name, str(langchain_asset_path))
        except (ValueError, AttributeError, NotImplementedError) as e:
            if hasattr(langchain_asset, "save_agent"):
                langchain_asset.save_agent(langchain_asset_path)
                self.experiment.log_model(model_name, str(langchain_asset_path))
            else:
                self.comet_ml.LOGGER.error(
                    f"{e}"
                    " Could not save Langchain Asset "
                    f"for {langchain_asset.__class__.__name__}"
                )
    def _log_session(self, langchain_asset: Optional[Any] = None) -> None:
        try:
            llm_session_df = self._create_session_analysis_dataframe(langchain_asset)
            # Log the cleaned dataframe as a table
            self.experiment.log_table("langchain-llm-session.csv", llm_session_df)
        except Exception:
            self.comet_ml.LOGGER.warning(
                "Failed to log session data to Comet",
                exc_info=True,
                extra={"show_traceback": True},
            )
exc_info=True,
                extra={"show_traceback": True},
            )
        try:
            metadata = {"langchain_version": str(langchain_community.__version__)}
            # Log the langchain low-level records as a JSON file directly
            self.experiment.log_asset_data(
                self.action_records, "langchain-action_records.json", metadata=metadata
            )
        except Exception:
            self.comet_ml.LOGGER.warning(
                "Failed to log session data to Comet",
                exc_info=True,
                extra={"show_traceback": True},
            )
        try:
            self._log_visualizations(llm_session_df)
        except Exception:
            self.comet_ml.LOGGER.warning(
                "Failed to log visualizations to Comet",
                exc_info=True,
                extra={"show_traceback": True},
            )
    def _log_text_metrics(self, metrics: Sequence[dict], step: int) -> None:
        if not metrics:
            return
        metrics_summary = _summarize_metrics_for_generated_outputs(metrics)
        for key, value in metrics_summary.items():
            self.experiment.log_metrics(value, prefix=key, step=step)
    def _log_visualizations(self, session_df: Any) -> None:
        if not (self.visualizations and self.nlp):
            return
        spacy = import_spacy()
        prompts = session_df["prompts"].tolist()
        outputs = session_df["text"].tolist()
        for idx, (prompt, output) in enumerate(zip(prompts, outputs)):
            doc = self.nlp(output)
            sentence_spans = list(doc.sents)
            for visualization in self.visualizations:
                try:
                    html = spacy.displacy.render(
                        sentence_spans,
html = spacy.displacy.render(
                        sentence_spans,
                        style=visualization,
                        options={"compact": True},
                        jupyter=False,
                        page=True,
                    )
                    self.experiment.log_asset_data(
                        html,
                        name=f"langchain-viz-{visualization}-{idx}.html",
                        metadata={"prompt": prompt},
                        step=idx,
                    )
                except Exception as e:
                    self.comet_ml.LOGGER.warning(
                        e, exc_info=True, extra={"show_traceback": True}
                    )
        return
    def _reset(
        self,
        task_type: Optional[str] = None,
        workspace: Optional[str] = None,
        project_name: Optional[str] = None,
        tags: Optional[Sequence] = None,
        name: Optional[str] = None,
        visualizations: Optional[List[str]] = None,
        complexity_metrics: bool = False,
        custom_metrics: Optional[Callable] = None,
    ) -> None:
        _task_type = task_type if task_type else self.task_type
        _workspace = workspace if workspace else self.workspace
        _project_name = project_name if project_name else self.project_name
        _tags = tags if tags else self.tags
        _name = name if name else self.name
        _visualizations = visualizations if visualizations else self.visualizations
        _complexity_metrics = (
            complexity_metrics if complexity_metrics else self.complexity_metrics
        )
        _custom_metrics = custom_metrics if custom_metrics else self.custom_metrics
        self.__init__(  # type: ignore
            task_type=_task_type,
            workspace=_workspace,
            project_name=_project_name,
            tags=_tags,
            name=_name,
project_name=_project_name,
            tags=_tags,
            name=_name,
            visualizations=_visualizations,
            complexity_metrics=_complexity_metrics,
            custom_metrics=_custom_metrics,
        )
        self.reset_callback_meta()
        self.temp_dir = tempfile.TemporaryDirectory()
    def _create_session_analysis_dataframe(self, langchain_asset: Any = None) -> dict:
        pd = import_pandas()
        llm_parameters = self._get_llm_parameters(langchain_asset)
        num_generations_per_prompt = llm_parameters.get("n", 1)
        llm_start_records_df = pd.DataFrame(self.on_llm_start_records)
        # Repeat each input row based on the number of outputs generated per prompt
        llm_start_records_df = llm_start_records_df.loc[
            llm_start_records_df.index.repeat(num_generations_per_prompt)
        ].reset_index(drop=True)
        llm_end_records_df = pd.DataFrame(self.on_llm_end_records)
        llm_session_df = pd.merge(
            llm_start_records_df,
            llm_end_records_df,
            left_index=True,
            right_index=True,
            suffixes=["_llm_start", "_llm_end"],
        )
        return llm_session_df
    def _get_llm_parameters(self, langchain_asset: Any = None) -> dict:
        if not langchain_asset:
            return {}
        try:
            if hasattr(langchain_asset, "agent"):
                llm_parameters = langchain_asset.agent.llm_chain.llm.dict()
            elif hasattr(langchain_asset, "llm_chain"):
                llm_parameters = langchain_asset.llm_chain.llm.dict()
            elif hasattr(langchain_asset, "llm"):
                llm_parameters = langchain_asset.llm.dict()
Source code for langchain_community.callbacks.whylabs_callback
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Optional
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.utils import get_from_env
if TYPE_CHECKING:
    from whylogs.api.logger.logger import Logger
diagnostic_logger = logging.getLogger(__name__)
[docs]def import_langkit(
    sentiment: bool = False,
    toxicity: bool = False,
    themes: bool = False,
) -> Any:
    """Import the langkit python package and raise an error if it is not installed.
    Args:
        sentiment: Whether to import the langkit.sentiment module. Defaults to False.
        toxicity: Whether to import the langkit.toxicity module. Defaults to False.
        themes: Whether to import the langkit.themes module. Defaults to False.
    Returns:
        The imported langkit module.
    """
    try:
        import langkit  # noqa: F401
        import langkit.regexes  # noqa: F401
        import langkit.textstat  # noqa: F401
        if sentiment:
            import langkit.sentiment  # noqa: F401
        if toxicity:
            import langkit.toxicity  # noqa: F401
        if themes:
            import langkit.themes  # noqa: F401
    except ImportError:
        raise ImportError(
            "To use the whylabs callback manager you need to have the `langkit` python "
            "package installed. Please install it with `pip install langkit`."
        )
    return langkit
[docs]class WhyLabsCallbackHandler(BaseCallbackHandler):
    """
    Callback Handler for logging to WhyLabs. This callback handler utilizes
"""
    Callback Handler for logging to WhyLabs. This callback handler utilizes
    `langkit` to extract features from the prompts & responses when interacting with
    an LLM. These features can be used to guardrail, evaluate, and observe interactions
    over time to detect issues relating to hallucinations, prompt engineering,
    or output validation. LangKit is an LLM monitoring toolkit developed by WhyLabs.
    Here are some examples of what can be monitored with LangKit:
    * Text Quality
      - readability score
      - complexity and grade scores
    * Text Relevance
      - Similarity scores between prompt/responses
      - Similarity scores against user-defined themes
      - Topic classification
    * Security and Privacy
      - patterns - count of strings matching a user-defined regex pattern group
      - jailbreaks - similarity scores with respect to known jailbreak attempts
      - prompt injection - similarity scores with respect to known prompt attacks
      - refusals - similarity scores with respect to known LLM refusal responses
    * Sentiment and Toxicity
      - sentiment analysis
      - toxicity analysis
    For more information, see https://docs.whylabs.ai/docs/language-model-monitoring
    or check out the LangKit repo here: https://github.com/whylabs/langkit
    ---
    Args:
        api_key (Optional[str]): WhyLabs API key. Optional because the preferred
            way to specify the API key is with environment variable
            WHYLABS_API_KEY.
        org_id (Optional[str]): WhyLabs organization id to write profiles to.
            Optional because the preferred way to specify the organization id is
            with environment variable WHYLABS_DEFAULT_ORG_ID.
        dataset_id (Optional[str]): WhyLabs dataset id to write profiles to.
            Optional because the preferred way to specify the dataset id is
Optional because the preferred way to specify the dataset id is
            with environment variable WHYLABS_DEFAULT_DATASET_ID.
        sentiment (bool): Whether to enable sentiment analysis. Defaults to False.
        toxicity (bool): Whether to enable toxicity analysis. Defaults to False.
        themes (bool): Whether to enable theme analysis. Defaults to False.
    """
[docs]    def __init__(self, logger: Logger, handler: Any):
        """Initiate the rolling logger."""
        super().__init__()
        if hasattr(handler, "init"):
            handler.init(self)
        if hasattr(handler, "_get_callbacks"):
            self._callbacks = handler._get_callbacks()
        else:
            self._callbacks = dict()
            diagnostic_logger.warning("initialized handler without callbacks.")
        self._logger = logger
[docs]    def flush(self) -> None:
        """Explicitly write current profile if using a rolling logger."""
        if self._logger and hasattr(self._logger, "_do_rollover"):
            self._logger._do_rollover()
            diagnostic_logger.info("Flushing WhyLabs logger, writing profile...")
[docs]    def close(self) -> None:
        """Close any loggers to allow writing out of any profiles before exiting."""
        if self._logger and hasattr(self._logger, "close"):
            self._logger.close()
            diagnostic_logger.info("Closing WhyLabs logger, see you next time!")
    def __enter__(self) -> WhyLabsCallbackHandler:
        return self
    def __exit__(
        self, exception_type: Any, exception_value: Any, traceback: Any
    ) -> None:
        self.close()
[docs]    @classmethod
    def from_params(
        cls,
        *,
[docs]    @classmethod
    def from_params(
        cls,
        *,
        api_key: Optional[str] = None,
        org_id: Optional[str] = None,
        dataset_id: Optional[str] = None,
        sentiment: bool = False,
        toxicity: bool = False,
        themes: bool = False,
        logger: Optional[Logger] = None,
    ) -> WhyLabsCallbackHandler:
        """Instantiate whylogs Logger from params.
        Args:
            api_key (Optional[str]): WhyLabs API key. Optional because the preferred
                way to specify the API key is with environment variable
                WHYLABS_API_KEY.
            org_id (Optional[str]): WhyLabs organization id to write profiles to.
                If not set must be specified in environment variable
                WHYLABS_DEFAULT_ORG_ID.
            dataset_id (Optional[str]): The model or dataset this callback is gathering
                telemetry for. If not set must be specified in environment variable
                WHYLABS_DEFAULT_DATASET_ID.
            sentiment (bool): If True will initialize a model to perform
                sentiment analysis compound score. Defaults to False and will not gather
                this metric.
            toxicity (bool): If True will initialize a model to score
                toxicity. Defaults to False and will not gather this metric.
            themes (bool): If True will initialize a model to calculate
                distance to configured themes. Defaults to None and will not gather this
                metric.
            logger (Optional[Logger]): If specified will bind the configured logger as
                the telemetry gathering agent. Defaults to LangKit schema with periodic
                WhyLabs writer.
        """
        # langkit library will import necessary whylogs libraries
        import_langkit(sentiment=sentiment, toxicity=toxicity, themes=themes)
import whylogs as why
        from langkit.callback_handler import get_callback_instance
        from whylogs.api.writer.whylabs import WhyLabsWriter
        from whylogs.experimental.core.udf_schema import udf_schema
        if logger is None:
            api_key = api_key or get_from_env("api_key", "WHYLABS_API_KEY")
            org_id = org_id or get_from_env("org_id", "WHYLABS_DEFAULT_ORG_ID")
            dataset_id = dataset_id or get_from_env(
                "dataset_id", "WHYLABS_DEFAULT_DATASET_ID"
            )
            whylabs_writer = WhyLabsWriter(
                api_key=api_key, org_id=org_id, dataset_id=dataset_id
            )
            whylabs_logger = why.logger(
                mode="rolling", interval=5, when="M", schema=udf_schema()
            )
            whylabs_logger.append_writer(writer=whylabs_writer)
        else:
            diagnostic_logger.info("Using passed in whylogs logger {logger}")
            whylabs_logger = logger
        callback_handler_cls = get_callback_instance(logger=whylabs_logger, impl=cls)
        diagnostic_logger.info(
            "Started whylogs Logger with WhyLabsWriter and initialized LangKit. 📝"
        )
        return callback_handler_cls
Source code for langchain_community.callbacks.manager
from __future__ import annotations
import logging
from contextlib import contextmanager
from contextvars import ContextVar
from typing import (
    Generator,
    Optional,
)
from langchain_core.tracers.context import register_configure_hook
from langchain_community.callbacks.openai_info import OpenAICallbackHandler
from langchain_community.callbacks.tracers.wandb import WandbTracer
logger = logging.getLogger(__name__)
openai_callback_var: ContextVar[Optional[OpenAICallbackHandler]] = ContextVar(
    "openai_callback", default=None
)
wandb_tracing_callback_var: ContextVar[Optional[WandbTracer]] = ContextVar(  # noqa: E501
    "tracing_wandb_callback", default=None
)
register_configure_hook(openai_callback_var, True)
register_configure_hook(
    wandb_tracing_callback_var, True, WandbTracer, "LANGCHAIN_WANDB_TRACING"
)
[docs]@contextmanager
def get_openai_callback() -> Generator[OpenAICallbackHandler, None, None]:
    """Get the OpenAI callback handler in a context manager.
    which conveniently exposes token and cost information.
    Returns:
        OpenAICallbackHandler: The OpenAI callback handler.
    Example:
        >>> with get_openai_callback() as cb:
        ...     # Use the OpenAI callback handler
    """
    cb = OpenAICallbackHandler()
    openai_callback_var.set(cb)
    yield cb
    openai_callback_var.set(None)
[docs]@contextmanager
def wandb_tracing_enabled(
    session_name: str = "default",
) -> Generator[None, None, None]:
    """Get the WandbTracer in a context manager.
    Args:
"""Get the WandbTracer in a context manager.
    Args:
        session_name (str, optional): The name of the session.
            Defaults to "default".
    Returns:
        None
    Example:
        >>> with wandb_tracing_enabled() as session:
        ...     # Use the WandbTracer session
    """
    cb = WandbTracer()
    wandb_tracing_callback_var.set(cb)
    yield None
    wandb_tracing_callback_var.set(None)
Source code for langchain_community.callbacks.labelstudio_callback
import os
import warnings
from datetime import datetime
from enum import Enum
from typing import Any, Dict, List, Optional, Tuple, Union
from uuid import UUID
from langchain_core.agents import AgentAction, AgentFinish
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.messages import BaseMessage, ChatMessage
from langchain_core.outputs import Generation, LLMResult
[docs]class LabelStudioMode(Enum):
    """Label Studio mode enumerator."""
    PROMPT = "prompt"
    CHAT = "chat"
[docs]def get_default_label_configs(
    mode: Union[str, LabelStudioMode],
) -> Tuple[str, LabelStudioMode]:
    """Get default Label Studio configs for the given mode.
    Parameters:
        mode: Label Studio mode ("prompt" or "chat")
    Returns: Tuple of Label Studio config and mode
    """
    _default_label_configs = {
        LabelStudioMode.PROMPT.value: """
<View>
<Style>
    .prompt-box {
        background-color: white;
        border-radius: 10px;
        box-shadow: 0px 4px 6px rgba(0, 0, 0, 0.1);
        padding: 20px;
    }
</Style>
<View className="root">
    <View className="prompt-box">
        <Text name="prompt" value="$prompt"/>
    </View>
    <TextArea name="response" toName="prompt"
              maxSubmissions="1" editable="true"
              required="true"/>
</View>
<Header value="Rate the response:"/>
<Rating name="rating" toName="prompt"/>
</View>""",
        LabelStudioMode.CHAT.value: """
</View>""",
        LabelStudioMode.CHAT.value: """
<View>
<View className="root">
     <Paragraphs name="dialogue"
               value="$prompt"
               layout="dialogue"
               textKey="content"
               nameKey="role"
               granularity="sentence"/>
  <Header value="Final response:"/>
    <TextArea name="response" toName="dialogue"
              maxSubmissions="1" editable="true"
              required="true"/>
</View>
<Header value="Rate the response:"/>
<Rating name="rating" toName="dialogue"/>
</View>""",
    }
    if isinstance(mode, str):
        mode = LabelStudioMode(mode)
    return _default_label_configs[mode.value], mode
[docs]class LabelStudioCallbackHandler(BaseCallbackHandler):
    """Label Studio callback handler.
    Provides the ability to send predictions to Label Studio
    for human evaluation, feedback and annotation.
    Parameters:
        api_key: Label Studio API key
        url: Label Studio URL
        project_id: Label Studio project ID
        project_name: Label Studio project name
        project_config: Label Studio project config (XML)
        mode: Label Studio mode ("prompt" or "chat")
    Examples:
        >>> from langchain_community.llms import OpenAI
        >>> from langchain_community.callbacks import LabelStudioCallbackHandler
        >>> handler = LabelStudioCallbackHandler(
        ...             api_key='<your_key_here>',
        ...             url='http://localhost:8080',
        ...             project_name='LangChain-%Y-%m-%d',
        ...             mode='prompt'
        ... )
        >>> llm = OpenAI(callbacks=[handler])
        >>> llm.predict('Tell me a story about a dog.')
>>> llm.predict('Tell me a story about a dog.')
    """
    DEFAULT_PROJECT_NAME: str = "LangChain-%Y-%m-%d"
[docs]    def __init__(
        self,
        api_key: Optional[str] = None,
        url: Optional[str] = None,
        project_id: Optional[int] = None,
        project_name: str = DEFAULT_PROJECT_NAME,
        project_config: Optional[str] = None,
        mode: Union[str, LabelStudioMode] = LabelStudioMode.PROMPT,
    ):
        super().__init__()
        # Import LabelStudio SDK
        try:
            import label_studio_sdk as ls
        except ImportError:
            raise ImportError(
                f"You're using {self.__class__.__name__} in your code,"
                f" but you don't have the LabelStudio SDK "
                f"Python package installed or upgraded to the latest version. "
                f"Please run `pip install -U label-studio-sdk`"
                f" before using this callback."
            )
        # Check if Label Studio API key is provided
        if not api_key:
            if os.getenv("LABEL_STUDIO_API_KEY"):
                api_key = str(os.getenv("LABEL_STUDIO_API_KEY"))
            else:
                raise ValueError(
                    f"You're using {self.__class__.__name__} in your code,"
                    f" Label Studio API key is not provided. "
                    f"Please provide Label Studio API key: "
                    f"go to the Label Studio instance, navigate to "
                    f"Account & Settings -> Access Token and copy the key. "
                    f"Use the key as a parameter for the callback: "
                    f"{self.__class__.__name__}"
f"{self.__class__.__name__}"
                    f"(label_studio_api_key='<your_key_here>', ...) or "
                    f"set the environment variable LABEL_STUDIO_API_KEY=<your_key_here>"
                )
        self.api_key = api_key
        if not url:
            if os.getenv("LABEL_STUDIO_URL"):
                url = os.getenv("LABEL_STUDIO_URL")
            else:
                warnings.warn(
                    f"Label Studio URL is not provided, "
                    f"using default URL: {ls.LABEL_STUDIO_DEFAULT_URL}"
                    f"If you want to provide your own URL, use the parameter: "
                    f"{self.__class__.__name__}"
                    f"(label_studio_url='<your_url_here>', ...) "
                    f"or set the environment variable LABEL_STUDIO_URL=<your_url_here>"
                )
                url = ls.LABEL_STUDIO_DEFAULT_URL
        self.url = url
        # Maps run_id to prompts
        self.payload: Dict[str, Dict] = {}
        self.ls_client = ls.Client(url=self.url, api_key=self.api_key)
        self.project_name = project_name
        if project_config:
            self.project_config = project_config
            self.mode = None
        else:
            self.project_config, self.mode = get_default_label_configs(mode)
        self.project_id = project_id or os.getenv("LABEL_STUDIO_PROJECT_ID")
        if self.project_id is not None:
            self.ls_project = self.ls_client.get_project(int(self.project_id))
        else:
            project_title = datetime.today().strftime(self.project_name)
            existing_projects = self.ls_client.get_projects(title=project_title)
            if existing_projects:
                self.ls_project = existing_projects[0]
                self.project_id = self.ls_project.id
            else: