viruPVS/langchain-docs huggingface dataset:
Source code for langchain_experimental.cpal.base
"""
CPAL Chain and its subchains
"""
from __future__ import annotations
import json
from typing import Any, ClassVar, Dict, List, Optional, Type
from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.output_parsers import PydanticOutputParser
from langchain.prompts.prompt import PromptTemplate
from langchain_experimental import pydantic_v1 as pydantic
from langchain_experimental.cpal.constants import Constant
from langchain_experimental.cpal.models import (
    CausalModel,
    InterventionModel,
    NarrativeModel,
    QueryModel,
    StoryModel,
)
from langchain_experimental.cpal.templates.univariate.causal import (
    template as causal_template,
)
from langchain_experimental.cpal.templates.univariate.intervention import (
    template as intervention_template,
)
from langchain_experimental.cpal.templates.univariate.narrative import (
    template as narrative_template,
)
from langchain_experimental.cpal.templates.univariate.query import (
    template as query_template,
)
class _BaseStoryElementChain(Chain):
    chain: LLMChain
    input_key: str = Constant.narrative_input.value  #: :meta private:
    output_key: str = Constant.chain_answer.value  #: :meta private:
    pydantic_model: ClassVar[
        Optional[Type[pydantic.BaseModel]]
    ] = None  #: :meta private:
    template: ClassVar[Optional[str]] = None  #: :meta private:
    @classmethod
    def parser(cls) -> PydanticOutputParser:
@classmethod
    def parser(cls) -> PydanticOutputParser:
        """Parse LLM output into a pydantic object."""
        if cls.pydantic_model is None:
            raise NotImplementedError(
                f"pydantic_model not implemented for {cls.__name__}"
            )
        return PydanticOutputParser(pydantic_object=cls.pydantic_model)
    @property
    def input_keys(self) -> List[str]:
        """Return the input keys.
        :meta private:
        """
        return [self.input_key]
    @property
    def output_keys(self) -> List[str]:
        """Return the output keys.
        :meta private:
        """
        _output_keys = [self.output_key]
        return _output_keys
    @classmethod
    def from_univariate_prompt(
        cls,
        llm: BaseLanguageModel,
        **kwargs: Any,
    ) -> Any:
        return cls(
            chain=LLMChain(
                llm=llm,
                prompt=PromptTemplate(
                    input_variables=[Constant.narrative_input.value],
                    template=kwargs.get("template", cls.template),
                    partial_variables={
                        "format_instructions": cls.parser().get_format_instructions()
                    },
                ),
            ),
            **kwargs,
        )
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        completion = self.chain.run(inputs[self.input_key])
        pydantic_data = self.__class__.parser().parse(completion)
        return {
            Constant.chain_data.value: pydantic_data,
            Constant.chain_answer.value: None,
        }
Constant.chain_answer.value: None,
        }
[docs]class NarrativeChain(_BaseStoryElementChain):
    """Decompose the narrative into its story elements
    - causal model
    - query
    - intervention
    """
    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = NarrativeModel
    template: ClassVar[str] = narrative_template
[docs]class CausalChain(_BaseStoryElementChain):
    """Translate the causal narrative into a stack of operations."""
    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = CausalModel
    template: ClassVar[str] = causal_template
[docs]class InterventionChain(_BaseStoryElementChain):
    """Set the hypothetical conditions for the causal model."""
    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = InterventionModel
    template: ClassVar[str] = intervention_template
[docs]class QueryChain(_BaseStoryElementChain):
    """Query the outcome table using SQL.
    *Security note*: This class implements an AI technique that generates SQL code.
        If those SQL commands are executed, it's critical to ensure they use credentials
        that are narrowly-scoped to only include the permissions this chain needs.
        Failure to do so may result in data corruption or loss, since this chain may
        attempt commands like `DROP TABLE` or `INSERT` if appropriately prompted.
        The best way to guard against such negative outcomes is to (as appropriate)
        limit the permissions granted to the credentials used with this chain.
    """
    pydantic_model: ClassVar[Type[pydantic.BaseModel]] = QueryModel
    template: ClassVar[str] = query_template  # TODO: incl. table schema
[docs]class CPALChain(_BaseStoryElementChain):
[docs]class CPALChain(_BaseStoryElementChain):
    """Causal program-aided language (CPAL) chain implementation.
    *Security note*: The building blocks of this class include the implementation
        of an AI technique that generates SQL code. If those SQL commands
        are executed, it's critical to ensure they use credentials that
        are narrowly-scoped to only include the permissions this chain needs.
        Failure to do so may result in data corruption or loss, since this chain may
        attempt commands like `DROP TABLE` or `INSERT` if appropriately prompted.
        The best way to guard against such negative outcomes is to (as appropriate)
        limit the permissions granted to the credentials used with this chain.
    """
    llm: BaseLanguageModel
    narrative_chain: Optional[NarrativeChain] = None
    causal_chain: Optional[CausalChain] = None
    intervention_chain: Optional[InterventionChain] = None
    query_chain: Optional[QueryChain] = None
    _story: StoryModel = pydantic.PrivateAttr(default=None)  # TODO: change name ?
[docs]    @classmethod
    def from_univariate_prompt(
        cls,
        llm: BaseLanguageModel,
        **kwargs: Any,
    ) -> CPALChain:
        """instantiation depends on component chains
        *Security note*: The building blocks of this class include the implementation
            of an AI technique that generates SQL code. If those SQL commands
            are executed, it's critical to ensure they use credentials that
            are narrowly-scoped to only include the permissions this chain needs.
            Failure to do so may result in data corruption or loss, since this chain may
            attempt commands like `DROP TABLE` or `INSERT` if appropriately prompted.
attempt commands like `DROP TABLE` or `INSERT` if appropriately prompted.
            The best way to guard against such negative outcomes is to (as appropriate)
            limit the permissions granted to the credentials used with this chain.
        """
        return cls(
            llm=llm,
            chain=LLMChain(
                llm=llm,
                prompt=PromptTemplate(
                    input_variables=["question", "query_result"],
                    template=(
                        "Summarize this answer '{query_result}' to this "
                        "question '{question}'? "
                    ),
                ),
            ),
            narrative_chain=NarrativeChain.from_univariate_prompt(llm=llm),
            causal_chain=CausalChain.from_univariate_prompt(llm=llm),
            intervention_chain=InterventionChain.from_univariate_prompt(llm=llm),
            query_chain=QueryChain.from_univariate_prompt(llm=llm),
            **kwargs,
        )
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
        **kwargs: Any,
    ) -> Dict[str, Any]:
        # instantiate component chains
        if self.narrative_chain is None:
            self.narrative_chain = NarrativeChain.from_univariate_prompt(llm=self.llm)
        if self.causal_chain is None:
            self.causal_chain = CausalChain.from_univariate_prompt(llm=self.llm)
        if self.intervention_chain is None:
            self.intervention_chain = InterventionChain.from_univariate_prompt(
                llm=self.llm
            )
        if self.query_chain is None:
            self.query_chain = QueryChain.from_univariate_prompt(llm=self.llm)
self.query_chain = QueryChain.from_univariate_prompt(llm=self.llm)
        # decompose narrative into three causal story elements
        narrative = self.narrative_chain(inputs[Constant.narrative_input.value])[
            Constant.chain_data.value
        ]
        story = StoryModel(
            causal_operations=self.causal_chain(narrative.story_plot)[
                Constant.chain_data.value
            ],
            intervention=self.intervention_chain(narrative.story_hypothetical)[
                Constant.chain_data.value
            ],
            query=self.query_chain(narrative.story_outcome_question)[
                Constant.chain_data.value
            ],
        )
        self._story = story
        def pretty_print_str(title: str, d: str) -> str:
            return title + "\n" + d
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(
            pretty_print_str("story outcome data", story._outcome_table.to_string()),
            color="green",
            end="\n\n",
            verbose=self.verbose,
        )
        def pretty_print_dict(title: str, d: dict) -> str:
            return title + "\n" + json.dumps(d, indent=4)
        _run_manager.on_text(
            pretty_print_dict("query data", story.query.dict()),
            color="blue",
            end="\n\n",
            verbose=self.verbose,
        )
        if story.query._result_table.empty:
            # prevent piping bad data into subsequent chains
            raise ValueError(
                (
                    "unanswerable, query and outcome are incoherent\n"
                    "\n"
                    "outcome:\n"
                    f"{story._outcome_table}\n"
                    "query:\n"
                    f"{story.query.dict()}"
                )
"query:\n"
                    f"{story.query.dict()}"
                )
            )
        else:
            query_result = float(story.query._result_table.values[0][-1])
            if False:
                """TODO: add this back in when demanded by composable chains"""
                reporting_chain = self.chain
                human_report = reporting_chain.run(
                    question=story.query.question, query_result=query_result
                )
                query_result = {
                    "query_result": query_result,
                    "human_report": human_report,
                }
        output = {
            Constant.chain_data.value: story,
            self.output_key: query_result,
            **kwargs,
        }
        return output
[docs]    def draw(self, **kwargs: Any) -> None:
        """
        CPAL chain can draw its resulting DAG.
        Usage in a jupyter notebook:
            >>> from IPython.display import SVG
            >>> cpal_chain.draw(path="graph.svg")
            >>> SVG('graph.svg')
        """
        self._story._networkx_wrapper.draw_graphviz(**kwargs)
Source code for langchain_experimental.cpal.models
from __future__ import annotations  # allows pydantic model to reference itself
import re
from typing import Any, List, Optional, Union
from langchain.graphs.networkx_graph import NetworkxEntityGraph
from langchain_experimental.cpal.constants import Constant
from langchain_experimental.pydantic_v1 import (
    BaseModel,
    Field,
    PrivateAttr,
    root_validator,
    validator,
)
[docs]class NarrativeModel(BaseModel):
    """
    Represent the narrative input as three story elements.
    """
    story_outcome_question: str
    story_hypothetical: str
    story_plot: str  # causal stack of operations
    @validator("*", pre=True)
    def empty_str_to_none(cls, v: str) -> Union[str, None]:
        """Empty strings are not allowed"""
        if v == "":
            return None
        return v
[docs]class EntityModel(BaseModel):
    name: str = Field(description="entity name")
    code: str = Field(description="entity actions")
    value: float = Field(description="entity initial value")
    depends_on: List[str] = Field(default=[], description="ancestor entities")
    # TODO: generalize to multivariate math
    # TODO: acyclic graph
    class Config:
        validate_assignment = True
    @validator("name")
    def lower_case_name(cls, v: str) -> str:
        v = v.lower()
        return v
[docs]class CausalModel(BaseModel):
    attribute: str = Field(description="name of the attribute to be calculated")
    entities: List[EntityModel] = Field(description="entities in the story")
entities: List[EntityModel] = Field(description="entities in the story")
    # TODO: root validate each `entity.depends_on` using system's entity names
[docs]class EntitySettingModel(BaseModel):
    """
    Initial conditions for an entity
    {"name": "bud", "attribute": "pet_count", "value": 12}
    """
    name: str = Field(description="name of the entity")
    attribute: str = Field(description="name of the attribute to be calculated")
    value: float = Field(description="entity's attribute value (calculated)")
    @validator("name")
    def lower_case_transform(cls, v: str) -> str:
        v = v.lower()
        return v
[docs]class SystemSettingModel(BaseModel):
    """
    Initial global conditions for the system.
    {"parameter": "interest_rate", "value": .05}
    """
    parameter: str
    value: float
[docs]class InterventionModel(BaseModel):
    """
    aka initial conditions
    >>> intervention.dict()
    {
        entity_settings: [
            {"name": "bud", "attribute": "pet_count", "value": 12},
            {"name": "pat", "attribute": "pet_count", "value": 0},
        ],
        system_settings: None,
    }
    """
    entity_settings: List[EntitySettingModel]
    system_settings: Optional[List[SystemSettingModel]] = None
    @validator("system_settings")
    def lower_case_name(cls, v: str) -> Union[str, None]:
        if v is not None:
            raise NotImplementedError("system_setting is not implemented yet")
        return v
[docs]class QueryModel(BaseModel):
return v
[docs]class QueryModel(BaseModel):
    """translate a question about the story outcome into a programmatic expression"""
    question: str = Field(alias=Constant.narrative_input.value)  # input
    expression: str  # output, part of llm completion
    llm_error_msg: str  # output, part of llm completion
    _result_table: str = PrivateAttr()  # result of the executed query
[docs]class ResultModel(BaseModel):
    question: str = Field(alias=Constant.narrative_input.value)  # input
    _result_table: str = PrivateAttr()  # result of the executed query
[docs]class StoryModel(BaseModel):
    causal_operations: Any = Field(required=True)
    intervention: Any = Field(required=True)
    query: Any = Field(required=True)
    _outcome_table: Any = PrivateAttr(default=None)
    _networkx_wrapper: Any = PrivateAttr(default=None)
    def __init__(self, **kwargs: Any):
        super().__init__(**kwargs)
        self._compute()
        # TODO: when langchain adopts pydantic.v2 replace w/ `__post_init__`
        # misses hints github.com/pydantic/pydantic/issues/1729#issuecomment-1300576214
    # TODO: move away from `root_validator` since it is deprecated in pydantic v2
    #       and causes mypy type-checking failures (hence the `type: ignore`)
    @root_validator  # type: ignore[call-overload]
    def check_intervention_is_valid(cls, values: dict) -> dict:
        valid_names = [e.name for e in values["causal_operations"].entities]
        for setting in values["intervention"].entity_settings:
for setting in values["intervention"].entity_settings:
            if setting.name not in valid_names:
                error_msg = f"""
                    Hypothetical question has an invalid entity name.
                    `{setting.name}` not in `{valid_names}`
                """
                raise ValueError(error_msg)
        return values
    def _block_back_door_paths(self) -> None:
        # stop intervention entities from depending on others
        intervention_entities = [
            entity_setting.name for entity_setting in self.intervention.entity_settings
        ]
        for entity in self.causal_operations.entities:
            if entity.name in intervention_entities:
                entity.depends_on = []
                entity.code = "pass"
    def _set_initial_conditions(self) -> None:
        for entity_setting in self.intervention.entity_settings:
            for entity in self.causal_operations.entities:
                if entity.name == entity_setting.name:
                    entity.value = entity_setting.value
    def _make_graph(self) -> None:
        self._networkx_wrapper = NetworkxEntityGraph()
        for entity in self.causal_operations.entities:
            for parent_name in entity.depends_on:
                self._networkx_wrapper._graph.add_edge(
                    parent_name, entity.name, relation=entity.code
                )
        # TODO: is it correct to drop entities with no impact on the outcome (?)
        self.causal_operations.entities = [
            entity
            for entity in self.causal_operations.entities
            if entity.name in self._networkx_wrapper.get_topological_sort()
        ]
    def _sort_entities(self) -> None:
        # order the sequence of causal actions
        sorted_nodes = self._networkx_wrapper.get_topological_sort()
        self.causal_operations.entities.sort(key=lambda x: sorted_nodes.index(x.name))
    def _forward_propagate(self) -> None:
def _forward_propagate(self) -> None:
        try:
            import pandas as pd
        except ImportError as e:
            raise ImportError(
                "Unable to import pandas, please install with `pip install pandas`."
            ) from e
        entity_scope = {
            entity.name: entity for entity in self.causal_operations.entities
        }
        for entity in self.causal_operations.entities:
            if entity.code == "pass":
                continue
            else:
                # gist.github.com/dean0x7d/df5ce97e4a1a05be4d56d1378726ff92
                exec(entity.code, globals(), entity_scope)
        row_values = [entity.dict() for entity in entity_scope.values()]
        self._outcome_table = pd.DataFrame(row_values)
    def _run_query(self) -> None:
        def humanize_sql_error_msg(error: str) -> str:
            pattern = r"column\s+(.*?)\s+not found"
            col_match = re.search(pattern, error)
            if col_match:
                return (
                    "SQL error: "
                    + col_match.group(1)
                    + " is not an attribute in your story!"
                )
            else:
                return str(error)
        if self.query.llm_error_msg == "":
            try:
                import duckdb
                df = self._outcome_table  # noqa
                query_result = duckdb.sql(self.query.expression).df()
                self.query._result_table = query_result
            except duckdb.BinderException as e:
                self.query._result_table = humanize_sql_error_msg(str(e))
            except ImportError as e:
                raise ImportError(
                    "Unable to import duckdb, please install with `pip install duckdb`."
"Unable to import duckdb, please install with `pip install duckdb`."
                ) from e
            except Exception as e:
                self.query._result_table = str(e)
        else:
            msg = "LLM maybe failed to translate question to SQL query."
            raise ValueError(
                {
                    "question": self.query.question,
                    "llm_error_msg": self.query.llm_error_msg,
                    "msg": msg,
                }
            )
    def _compute(self) -> Any:
        self._block_back_door_paths()
        self._set_initial_conditions()
        self._make_graph()
        self._sort_entities()
        self._forward_propagate()
        self._run_query()
[docs]    def print_debug_report(self) -> None:
        report = {
            "outcome": self._outcome_table,
            "query": self.query.dict(),
            "result": self.query._result_table,
        }
        from pprint import pprint
        pprint(report)
Source code for langchain_experimental.comprehend_moderation.base_moderation_config
from typing import List, Union
from pydantic import BaseModel
[docs]class ModerationPiiConfig(BaseModel):
    threshold: float = 0.5
    """Threshold for PII confidence score, defaults to 0.5 i.e. 50%"""
    labels: List[str] = []
    """
    List of PII Universal Labels. 
    Defaults to `list[]`
    """
    redact: bool = False
    """Whether to perform redaction of detected PII entities"""
    mask_character: str = "*"
    """Redaction mask character in case redact=True, defaults to asterisk (*)"""
[docs]class ModerationToxicityConfig(BaseModel):
    threshold: float = 0.5
    """Threshold for Toxic label confidence score, defaults to 0.5 i.e. 50%"""
    labels: List[str] = []
    """List of toxic labels, defaults to `list[]`"""
[docs]class ModerationPromptSafetyConfig(BaseModel):
    threshold: float = 0.5
    """
    Threshold for Prompt Safety classification
    confidence score, defaults to 0.5 i.e. 50%
    """
[docs]class BaseModerationConfig(BaseModel):
    filters: List[
        Union[
            ModerationPiiConfig, ModerationToxicityConfig, ModerationPromptSafetyConfig
        ]
    ] = [
        ModerationPiiConfig(),
        ModerationToxicityConfig(),
        ModerationPromptSafetyConfig(),
    ]
    """
    Filters applied to the moderation chain, defaults to
    `[ModerationPiiConfig(), ModerationToxicityConfig(),
Source code for langchain_experimental.comprehend_moderation.pii
import asyncio
from typing import Any, Dict, Optional
from langchain_experimental.comprehend_moderation.base_moderation_exceptions import (
    ModerationPiiError,
)
[docs]class ComprehendPII:
[docs]    def __init__(
        self,
        client: Any,
        callback: Optional[Any] = None,
        unique_id: Optional[str] = None,
        chain_id: Optional[str] = None,
    ) -> None:
        self.client = client
        self.moderation_beacon = {
            "moderation_chain_id": chain_id,
            "moderation_type": "PII",
            "moderation_status": "LABELS_NOT_FOUND",
        }
        self.callback = callback
        self.unique_id = unique_id
[docs]    def validate(self, prompt_value: str, config: Any = None) -> str:
        redact = config.get("redact")
        return (
            self._detect_pii(prompt_value=prompt_value, config=config)
            if redact
            else self._contains_pii(prompt_value=prompt_value, config=config)
        )
    def _contains_pii(self, prompt_value: str, config: Any = None) -> str:
        """
        Checks for Personally Identifiable Information (PII) labels above a
        specified threshold. Uses Amazon Comprehend Contains PII Entities API. See -
        https://docs.aws.amazon.com/comprehend/latest/APIReference/API_ContainsPiiEntities.html
        Args:
            prompt_value (str): The input text to be checked for PII labels.
            config (Dict[str, Any]): Configuration for PII check and actions.
        Returns: