viruPVS/langchain-docs huggingface dataset:
analyzer_results=analyzer_results,
            operators=self.operators,
        )
        new_deanonymizer_mapping = create_anonymizer_mapping(
            text,
            filtered_analyzer_results,
            anonymizer_results,
            is_reversed=True,
        )
        self._deanonymizer_mapping.update(new_deanonymizer_mapping)
        return exact_matching_strategy(text, self.anonymizer_mapping)
    def _deanonymize(
        self,
        text_to_deanonymize: str,
        deanonymizer_matching_strategy: Callable[
            [str, MappingDataType], str
        ] = DEFAULT_DEANONYMIZER_MATCHING_STRATEGY,
    ) -> str:
        """Deanonymize text.
        Each anonymized entity is replaced with its original value.
        This method exploits the mapping created during the anonymization process.
        Args:
            text_to_deanonymize: text to deanonymize
            deanonymizer_matching_strategy: function to use to match
                anonymized entities with their original values and replace them.
        """
        if not self._deanonymizer_mapping:
            raise ValueError(
                "Deanonymizer mapping is empty.",
                "Please call anonymize() and anonymize some text first.",
            )
        text_to_deanonymize = deanonymizer_matching_strategy(
            text_to_deanonymize, self.deanonymizer_mapping
        )
        return text_to_deanonymize
[docs]    def reset_deanonymizer_mapping(self) -> None:
        """Reset the deanonymizer mapping"""
        self._deanonymizer_mapping = DeanonymizerMapping()
[docs]    def save_deanonymizer_mapping(self, file_path: Union[Path, str]) -> None:
"""Save the deanonymizer mapping to a JSON or YAML file.
        Args:
            file_path: Path to file to save the mapping to.
        Example:
        .. code-block:: python
            anonymizer.save_deanonymizer_mapping(file_path="path/mapping.json")
        """
        save_path = Path(file_path)
        if save_path.suffix not in [".json", ".yaml"]:
            raise ValueError(f"{save_path} must have an extension of .json or .yaml")
        # Make sure parent directories exist
        save_path.parent.mkdir(parents=True, exist_ok=True)
        if save_path.suffix == ".json":
            with open(save_path, "w") as f:
                json.dump(self.deanonymizer_mapping, f, indent=2)
        elif save_path.suffix == ".yaml":
            with open(save_path, "w") as f:
                yaml.dump(self.deanonymizer_mapping, f, default_flow_style=False)
[docs]    def load_deanonymizer_mapping(self, file_path: Union[Path, str]) -> None:
        """Load the deanonymizer mapping from a JSON or YAML file.
        Args:
            file_path: Path to file to load the mapping from.
        Example:
        .. code-block:: python
            anonymizer.load_deanonymizer_mapping(file_path="path/mapping.json")
        """
        load_path = Path(file_path)
        if load_path.suffix not in [".json", ".yaml"]:
            raise ValueError(f"{load_path} must have an extension of .json or .yaml")
        if load_path.suffix == ".json":
            with open(load_path, "r") as f:
                loaded_mapping = json.load(f)
        elif load_path.suffix == ".yaml":
Source code for langchain_experimental.data_anonymizer.faker_presidio_mapping
import string
from typing import Callable, Dict, Optional
[docs]def get_pseudoanonymizer_mapping(seed: Optional[int] = None) -> Dict[str, Callable]:
    try:
        from faker import Faker
    except ImportError as e:
        raise ImportError(
            "Could not import faker, please install it with `pip install Faker`."
        ) from e
    fake = Faker()
    fake.seed_instance(seed)
    # Listed  entities supported by Microsoft Presidio (for now, global and US only)
    # Source: https://microsoft.github.io/presidio/supported_entities/
    return {
        # Global entities
        "PERSON": lambda _: fake.name(),
        "EMAIL_ADDRESS": lambda _: fake.email(),
        "PHONE_NUMBER": lambda _: fake.phone_number(),
        "IBAN_CODE": lambda _: fake.iban(),
        "CREDIT_CARD": lambda _: fake.credit_card_number(),
        "CRYPTO": lambda _: "bc1"
        + "".join(
            fake.random_choices(string.ascii_lowercase + string.digits, length=26)
        ),
        "IP_ADDRESS": lambda _: fake.ipv4_public(),
        "LOCATION": lambda _: fake.city(),
        "DATE_TIME": lambda _: fake.date(),
        "NRP": lambda _: str(fake.random_number(digits=8, fix_len=True)),
        "MEDICAL_LICENSE": lambda _: fake.bothify(text="??######").upper(),
        "URL": lambda _: fake.url(),
        # US-specific entities
        "US_BANK_NUMBER": lambda _: fake.bban(),
        "US_DRIVER_LICENSE": lambda _: str(fake.random_number(digits=9, fix_len=True)),
"US_ITIN": lambda _: fake.bothify(text="9##-7#-####"),
        "US_PASSPORT": lambda _: fake.bothify(text="#####??").upper(),
        "US_SSN": lambda _: fake.ssn(),
        # UK-specific entities
        "UK_NHS": lambda _: str(fake.random_number(digits=10, fix_len=True)),
        # Spain-specific entities
        "ES_NIF": lambda _: fake.bothify(text="########?").upper(),
        # Italy-specific entities
        "IT_FISCAL_CODE": lambda _: fake.bothify(text="??????##?##?###?").upper(),
        "IT_DRIVER_LICENSE": lambda _: fake.bothify(text="?A#######?").upper(),
        "IT_VAT_CODE": lambda _: fake.bothify(text="IT???????????"),
        "IT_PASSPORT": lambda _: str(fake.random_number(digits=9, fix_len=True)),
        "IT_IDENTITY_CARD": lambda _: lambda _: str(
            fake.random_number(digits=7, fix_len=True)
        ),
        # Singapore-specific entities
        "SG_NRIC_FIN": lambda _: fake.bothify(text="????####?").upper(),
        # Australia-specific entities
        "AU_ABN": lambda _: str(fake.random_number(digits=11, fix_len=True)),
        "AU_ACN": lambda _: str(fake.random_number(digits=9, fix_len=True)),
        "AU_TFN": lambda _: str(fake.random_number(digits=9, fix_len=True)),
        "AU_MEDICARE": lambda _: str(fake.random_number(digits=10, fix_len=True)),
    }
Source code for langchain_experimental.data_anonymizer.deanonymizer_mapping
import re
from collections import defaultdict
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Dict, List
if TYPE_CHECKING:
    from presidio_analyzer import RecognizerResult
    from presidio_anonymizer.entities import EngineResult
MappingDataType = Dict[str, Dict[str, str]]
[docs]def format_duplicated_operator(operator_name: str, count: int) -> str:
    """Format the operator name with the count"""
    clean_operator_name = re.sub(r"[<>]", "", operator_name)
    clean_operator_name = re.sub(r"_\d+$", "", clean_operator_name)
    if operator_name.startswith("<") and operator_name.endswith(">"):
        return f"<{clean_operator_name}_{count}>"
    else:
        return f"{clean_operator_name}_{count}"
[docs]@dataclass
class DeanonymizerMapping:
    mapping: MappingDataType = field(
        default_factory=lambda: defaultdict(lambda: defaultdict(str))
    )
    @property
    def data(self) -> MappingDataType:
        """Return the deanonymizer mapping"""
        return {k: dict(v) for k, v in self.mapping.items()}
[docs]    def update(self, new_mapping: MappingDataType) -> None:
        """Update the deanonymizer mapping with new values
        Duplicated values will not be added
        If there are multiple entities of the same type, the mapping will
        include a count to differentiate them. For example, if there are
        two names in the input text, the mapping will include NAME_1 and NAME_2.
        """
        seen_values = set()
        for entity_type, values in new_mapping.items():
seen_values = set()
        for entity_type, values in new_mapping.items():
            count = len(self.mapping[entity_type]) + 1
            for key, value in values.items():
                if (
                    value not in seen_values
                    and value not in self.mapping[entity_type].values()
                ):
                    new_key = (
                        format_duplicated_operator(key, count)
                        if key in self.mapping[entity_type]
                        else key
                    )
                    self.mapping[entity_type][new_key] = value
                    seen_values.add(value)
                    count += 1
[docs]def create_anonymizer_mapping(
    original_text: str,
    analyzer_results: List["RecognizerResult"],
    anonymizer_results: "EngineResult",
    is_reversed: bool = False,
) -> MappingDataType:
    """Creates or updates the mapping used to anonymize and/or deanonymize text.
    This method exploits the results returned by the
    analysis and anonymization processes.
    If is_reversed is True, it constructs a mapping from each original
    entity to its anonymized value.
    If is_reversed is False, it constructs a mapping from each
    anonymized entity back to its original text value.
    If there are multiple entities of the same type, the mapping will
    include a count to differentiate them. For example, if there are
    two names in the input text, the mapping will include NAME_1 and NAME_2.
    Example of mapping:
    {
        "PERSON": {
            "<original>": "<anonymized>",
            "John Doe": "Slim Shady"
        },
        "PHONE_NUMBER": {
            "111-111-1111": "555-555-5555"
        }
        ...
    }
}
        ...
    }
    """
    # We are able to zip and loop through both lists because we expect
    # them to return corresponding entities for each identified piece
    # of analyzable data from our input.
    # We sort them by their 'start' attribute because it allows us to
    # match corresponding entities by their position in the input text.
    analyzer_results.sort(key=lambda d: d.start)
    anonymizer_results.items.sort(key=lambda d: d.start)
    mapping: MappingDataType = defaultdict(dict)
    count: dict = defaultdict(int)
    for analyzed, anonymized in zip(analyzer_results, anonymizer_results.items):
        original_value = original_text[analyzed.start : analyzed.end]
        entity_type = anonymized.entity_type
        if is_reversed:
            cond = original_value in mapping[entity_type].values()
        else:
            cond = original_value in mapping[entity_type]
        if cond:
            continue
        if (
            anonymized.text in mapping[entity_type].values()
            or anonymized.text in mapping[entity_type]
        ):
            anonymized_value = format_duplicated_operator(
                anonymized.text, count[entity_type] + 2
            )
            count[entity_type] += 1
        else:
            anonymized_value = anonymized.text
        mapping_key, mapping_value = (
            (anonymized_value, original_value)
            if is_reversed
            else (original_value, anonymized_value)
        )
        mapping[entity_type][mapping_key] = mapping_value
    return mapping
Source code for langchain_experimental.data_anonymizer.deanonymizer_matching_strategies
import re
from typing import List
from langchain_experimental.data_anonymizer.deanonymizer_mapping import MappingDataType
[docs]def exact_matching_strategy(text: str, deanonymizer_mapping: MappingDataType) -> str:
    """
    Exact matching strategy for deanonymization.
    It replaces all the anonymized entities with the original ones.
    Args:
        text: text to deanonymize
        deanonymizer_mapping: mapping between anonymized entities and original ones"""
    # Iterate over all the entities (PERSON, EMAIL_ADDRESS, etc.)
    for entity_type in deanonymizer_mapping:
        for anonymized, original in deanonymizer_mapping[entity_type].items():
            text = text.replace(anonymized, original)
    return text
[docs]def case_insensitive_matching_strategy(
    text: str, deanonymizer_mapping: MappingDataType
) -> str:
    """
    Case insensitive matching strategy for deanonymization.
    It replaces all the anonymized entities with the original ones
        irrespective of their letter case.
    Args:
        text: text to deanonymize
        deanonymizer_mapping: mapping between anonymized entities and original ones
    Examples of matching:
        keanu reeves -> Keanu Reeves
        JOHN F. KENNEDY -> John F. Kennedy
    """
    # Iterate over all the entities (PERSON, EMAIL_ADDRESS, etc.)
    for entity_type in deanonymizer_mapping:
        for anonymized, original in deanonymizer_mapping[entity_type].items():
            # Use regular expressions for case-insensitive matching and replacing
            text = re.sub(anonymized, original, text, flags=re.IGNORECASE)
    return text
return text
[docs]def fuzzy_matching_strategy(
    text: str, deanonymizer_mapping: MappingDataType, max_l_dist: int = 3
) -> str:
    """
    Fuzzy matching strategy for deanonymization.
    It uses fuzzy matching to find the position of the anonymized entity in the text.
    It replaces all the anonymized entities with the original ones.
    Args:
        text: text to deanonymize
        deanonymizer_mapping: mapping between anonymized entities and original ones
        max_l_dist: maximum Levenshtein distance between the anonymized entity and the
            text segment to consider it a match
    Examples of matching:
        Kaenu Reves -> Keanu Reeves
        John F. Kennedy -> John Kennedy
    """
    try:
        from fuzzysearch import find_near_matches
    except ImportError as e:
        raise ImportError(
            "Could not import fuzzysearch, please install with "
            "`pip install fuzzysearch`."
        ) from e
    for entity_type in deanonymizer_mapping:
        for anonymized, original in deanonymizer_mapping[entity_type].items():
            matches = find_near_matches(anonymized, text, max_l_dist=max_l_dist)
            new_text = ""
            last_end = 0
            for m in matches:
                # add the text that isn't part of a match
                new_text += text[last_end : m.start]
                # add the replacement text
                new_text += original
                last_end = m.end
            # add the remaining text that wasn't part of a match
            new_text += text[last_end:]
            text = new_text
    return text
[docs]def combined_exact_fuzzy_matching_strategy(
return text
[docs]def combined_exact_fuzzy_matching_strategy(
    text: str, deanonymizer_mapping: MappingDataType, max_l_dist: int = 3
) -> str:
    """
    RECOMMENDED STRATEGY.
    Combined exact and fuzzy matching strategy for deanonymization.
    Args:
        text: text to deanonymize
        deanonymizer_mapping: mapping between anonymized entities and original ones
        max_l_dist: maximum Levenshtein distance between the anonymized entity and the
            text segment to consider it a match
    Examples of matching:
        Kaenu Reves -> Keanu Reeves
        John F. Kennedy -> John Kennedy
    """
    text = exact_matching_strategy(text, deanonymizer_mapping)
    text = fuzzy_matching_strategy(text, deanonymizer_mapping, max_l_dist)
    return text
[docs]def ngram_fuzzy_matching_strategy(
    text: str,
    deanonymizer_mapping: MappingDataType,
    fuzzy_threshold: int = 85,
    use_variable_length: bool = True,
) -> str:
    """
    N-gram fuzzy matching strategy for deanonymization.
    It replaces all the anonymized entities with the original ones.
    It uses fuzzy matching to find the position of the anonymized entity in the text.
    It generates n-grams of the same length as the anonymized entity from the text and
    uses fuzzy matching to find the position of the anonymized entity in the text.
    Args:
        text: text to deanonymize
        deanonymizer_mapping: mapping between anonymized entities and original ones
        fuzzy_threshold: fuzzy matching threshold
        use_variable_length: whether to use (n-1, n, n+1)-grams or just n-grams
    """
"""
    def generate_ngrams(words_list: List[str], n: int) -> list:
        """Generate n-grams from a list of words"""
        return [
            " ".join(words_list[i : i + n]) for i in range(len(words_list) - (n - 1))
        ]
    try:
        from fuzzywuzzy import fuzz
    except ImportError as e:
        raise ImportError(
            "Could not import fuzzywuzzy, please install with "
            "`pip install fuzzywuzzy`."
        ) from e
    text_words = text.split()
    replacements = []
    matched_indices: List[int] = []
    for entity_type in deanonymizer_mapping:
        for anonymized, original in deanonymizer_mapping[entity_type].items():
            anonymized_words = anonymized.split()
            if use_variable_length:
                gram_lengths = [
                    len(anonymized_words) - 1,
                    len(anonymized_words),
                    len(anonymized_words) + 1,
                ]
            else:
                gram_lengths = [len(anonymized_words)]
            for n in gram_lengths:
                if n > 0:  # Take only positive values
                    segments = generate_ngrams(text_words, n)
                    for i, segment in enumerate(segments):
                        if (
                            fuzz.ratio(anonymized.lower(), segment.lower())
                            > fuzzy_threshold
                            and i not in matched_indices
                        ):
                            replacements.append((i, n, original))
                            # Add the matched segment indices to the list
                            matched_indices.extend(range(i, i + n))
    # Sort replacements by index in reverse order
    replacements.sort(key=lambda x: x[0], reverse=True)
    # Apply replacements in reverse order to not affect subsequent indices
Source code for langchain_experimental.rl_chain.base
from __future__ import annotations
import logging
import os
from abc import ABC, abstractmethod
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Generic,
    List,
    Optional,
    Tuple,
    Type,
    TypeVar,
    Union,
)
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.prompts import (
    BasePromptTemplate,
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    SystemMessagePromptTemplate,
)
from langchain_experimental.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_experimental.rl_chain.metrics import (
    MetricsTrackerAverage,
    MetricsTrackerRollingWindow,
)
from langchain_experimental.rl_chain.model_repository import ModelRepository
from langchain_experimental.rl_chain.vw_logger import VwLogger
if TYPE_CHECKING:
    import vowpal_wabbit_next as vw
logger = logging.getLogger(__name__)
class _BasedOn:
    def __init__(self, value: Any):
        self.value = value
    def __str__(self) -> str:
        return str(self.value)
    __repr__ = __str__
[docs]def BasedOn(anything: Any) -> _BasedOn:
    return _BasedOn(anything)
class _ToSelectFrom:
    def __init__(self, value: Any):
        self.value = value
    def __str__(self) -> str:
        return str(self.value)
    __repr__ = __str__
[docs]def ToSelectFrom(anything: Any) -> _ToSelectFrom:
[docs]def ToSelectFrom(anything: Any) -> _ToSelectFrom:
    if not isinstance(anything, list):
        raise ValueError("ToSelectFrom must be a list to select from")
    return _ToSelectFrom(anything)
class _Embed:
    def __init__(self, value: Any, keep: bool = False):
        self.value = value
        self.keep = keep
    def __str__(self) -> str:
        return str(self.value)
    __repr__ = __str__
[docs]def Embed(anything: Any, keep: bool = False) -> Any:
    if isinstance(anything, _ToSelectFrom):
        return ToSelectFrom(Embed(anything.value, keep=keep))
    elif isinstance(anything, _BasedOn):
        return BasedOn(Embed(anything.value, keep=keep))
    if isinstance(anything, list):
        return [Embed(v, keep=keep) for v in anything]
    elif isinstance(anything, dict):
        return {k: Embed(v, keep=keep) for k, v in anything.items()}
    elif isinstance(anything, _Embed):
        return anything
    return _Embed(anything, keep=keep)
[docs]def EmbedAndKeep(anything: Any) -> Any:
    return Embed(anything, keep=True)
# helper functions
[docs]def stringify_embedding(embedding: List) -> str:
    return " ".join([f"{i}:{e}" for i, e in enumerate(embedding)])
[docs]def parse_lines(parser: "vw.TextFormatParser", input_str: str) -> List["vw.Example"]:
    return [parser.parse_line(line) for line in input_str.split("\n")]
return [parser.parse_line(line) for line in input_str.split("\n")]
[docs]def get_based_on_and_to_select_from(inputs: Dict[str, Any]) -> Tuple[Dict, Dict]:
    to_select_from = {
        k: inputs[k].value
        for k in inputs.keys()
        if isinstance(inputs[k], _ToSelectFrom)
    }
    if not to_select_from:
        raise ValueError(
            "No variables using 'ToSelectFrom' found in the inputs. Please include at least one variable containing a list to select from."  # noqa: E501
        )
    based_on = {
        k: inputs[k].value if isinstance(inputs[k].value, list) else [inputs[k].value]
        for k in inputs.keys()
        if isinstance(inputs[k], _BasedOn)
    }
    return based_on, to_select_from
[docs]def prepare_inputs_for_autoembed(inputs: Dict[str, Any]) -> Dict[str, Any]:
    """
    go over all the inputs and if something is either wrapped in _ToSelectFrom or _BasedOn, and if their inner values are not already _Embed,
    then wrap them in EmbedAndKeep while retaining their _ToSelectFrom or _BasedOn status
    """  # noqa: E501
    next_inputs = inputs.copy()
    for k, v in next_inputs.items():
        if isinstance(v, _ToSelectFrom) or isinstance(v, _BasedOn):
            if not isinstance(v.value, _Embed):
                next_inputs[k].value = EmbedAndKeep(v.value)
    return next_inputs
# end helper functions
[docs]class Selected(ABC):
    pass
TSelected = TypeVar("TSelected", bound=Selected)
[docs]class Event(Generic[TSelected], ABC):
[docs]class Event(Generic[TSelected], ABC):
    inputs: Dict[str, Any]
    selected: Optional[TSelected]
[docs]    def __init__(self, inputs: Dict[str, Any], selected: Optional[TSelected] = None):
        self.inputs = inputs
        self.selected = selected
TEvent = TypeVar("TEvent", bound=Event)
[docs]class Policy(Generic[TEvent], ABC):
[docs]    def __init__(self, **kwargs: Any):
        pass
[docs]    @abstractmethod
    def predict(self, event: TEvent) -> Any:
        ...
[docs]    @abstractmethod
    def learn(self, event: TEvent) -> None:
        ...
[docs]    @abstractmethod
    def log(self, event: TEvent) -> None:
        ...
[docs]    def save(self) -> None:
        pass
[docs]class VwPolicy(Policy):
[docs]    def __init__(
        self,
        model_repo: ModelRepository,
        vw_cmd: List[str],
        feature_embedder: Embedder,
        vw_logger: VwLogger,
        *args: Any,
        **kwargs: Any,
    ):
        super().__init__(*args, **kwargs)
        self.model_repo = model_repo
        self.workspace = self.model_repo.load(vw_cmd)
        self.feature_embedder = feature_embedder
        self.vw_logger = vw_logger
[docs]    def predict(self, event: TEvent) -> Any:
        import vowpal_wabbit_next as vw
        text_parser = vw.TextFormatParser(self.workspace)
        return self.workspace.predict_one(
            parse_lines(text_parser, self.feature_embedder.format(event))
        )