viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.document_transformers.html2text
from typing import Any, Sequence
from langchain_core.documents import BaseDocumentTransformer, Document
[docs]class Html2TextTransformer(BaseDocumentTransformer):
    """Replace occurrences of a particular search pattern with a replacement string
    Arguments:
        ignore_links: Whether links should be ignored; defaults to True.
        ignore_images: Whether images should be ignored; defaults to True.
    Example:
        .. code-block:: python
            from langchain_community.document_transformers import Html2TextTransformer
            html2text = Html2TextTransformer()
            docs_transform = html2text.transform_documents(docs)
    """
[docs]    def __init__(self, ignore_links: bool = True, ignore_images: bool = True) -> None:
        self.ignore_links = ignore_links
        self.ignore_images = ignore_images
[docs]    def transform_documents(
        self,
        documents: Sequence[Document],
        **kwargs: Any,
    ) -> Sequence[Document]:
        try:
            import html2text
        except ImportError:
            raise ImportError(
                """html2text package not found, please 
                install it with `pip install html2text`"""
            )
        # Create a html2text.HTML2Text object and override some properties
        h = html2text.HTML2Text()
        h.ignore_links = self.ignore_links
        h.ignore_images = self.ignore_images
        new_documents = []
        for d in documents:
            new_document = Document(
                page_content=h.handle(d.page_content), metadata={**d.metadata}
            )
            new_documents.append(new_document)
        return new_documents
[docs]    async def atransform_documents(
        self,
        documents: Sequence[Document],
        **kwargs: Any,
Source code for langchain_community.document_transformers.long_context_reorder
"""Reorder documents"""
from typing import Any, List, Sequence
from langchain_core.documents import BaseDocumentTransformer, Document
from langchain_core.pydantic_v1 import BaseModel
def _litm_reordering(documents: List[Document]) -> List[Document]:
    """Lost in the middle reorder: the less relevant documents will be at the
    middle of the list and more relevant elements at beginning / end.
    See: https://arxiv.org/abs//2307.03172"""
    documents.reverse()
    reordered_result = []
    for i, value in enumerate(documents):
        if i % 2 == 1:
            reordered_result.append(value)
        else:
            reordered_result.insert(0, value)
    return reordered_result
[docs]class LongContextReorder(BaseDocumentTransformer, BaseModel):
    """Lost in the middle:
    Performance degrades when models must access relevant information
    in the middle of long contexts.
    See: https://arxiv.org/abs//2307.03172"""
    class Config:
        """Configuration for this pydantic object."""
        arbitrary_types_allowed = True
[docs]    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Reorders documents."""
        return _litm_reordering(list(documents))
[docs]    async def atransform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        raise NotImplementedError
Source code for langchain_community.document_transformers.embeddings_redundant_filter
"""Transform documents"""
from typing import Any, Callable, List, Sequence
import numpy as np
from langchain_core.documents import BaseDocumentTransformer, Document
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Field
from langchain_community.utils.math import cosine_similarity
class _DocumentWithState(Document):
    """Wrapper for a document that includes arbitrary state."""
    state: dict = Field(default_factory=dict)
    """State associated with the document."""
    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    def to_document(self) -> Document:
        """Convert the DocumentWithState to a Document."""
        return Document(page_content=self.page_content, metadata=self.metadata)
    @classmethod
    def from_document(cls, doc: Document) -> "_DocumentWithState":
        """Create a DocumentWithState from a Document."""
        if isinstance(doc, cls):
            return doc
        return cls(page_content=doc.page_content, metadata=doc.metadata)
[docs]def get_stateful_documents(
    documents: Sequence[Document],
) -> Sequence[_DocumentWithState]:
    """Convert a list of documents to a list of documents with state.
    Args:
        documents: The documents to convert.
    Returns:
        A list of documents with state.
    """
    return [_DocumentWithState.from_document(doc) for doc in documents]
def _filter_similar_embeddings(
    embedded_documents: List[List[float]], similarity_fn: Callable, threshold: float
) -> List[int]:
    """Filter redundant documents based on the similarity of their embeddings."""
    similarity = np.tril(similarity_fn(embedded_documents, embedded_documents), k=-1)
redundant = np.where(similarity > threshold)
    redundant_stacked = np.column_stack(redundant)
    redundant_sorted = np.argsort(similarity[redundant])[::-1]
    included_idxs = set(range(len(embedded_documents)))
    for first_idx, second_idx in redundant_stacked[redundant_sorted]:
        if first_idx in included_idxs and second_idx in included_idxs:
            # Default to dropping the second document of any highly similar pair.
            included_idxs.remove(second_idx)
    return list(sorted(included_idxs))
def _get_embeddings_from_stateful_docs(
    embeddings: Embeddings, documents: Sequence[_DocumentWithState]
) -> List[List[float]]:
    if len(documents) and "embedded_doc" in documents[0].state:
        embedded_documents = [doc.state["embedded_doc"] for doc in documents]
    else:
        embedded_documents = embeddings.embed_documents(
            [d.page_content for d in documents]
        )
        for doc, embedding in zip(documents, embedded_documents):
            doc.state["embedded_doc"] = embedding
    return embedded_documents
def _filter_cluster_embeddings(
    embedded_documents: List[List[float]],
    num_clusters: int,
    num_closest: int,
    random_state: int,
    remove_duplicates: bool,
) -> List[int]:
    """Filter documents based on proximity of their embeddings to clusters."""
    try:
        from sklearn.cluster import KMeans
    except ImportError:
        raise ImportError(
            "sklearn package not found, please install it with "
            "`pip install scikit-learn`"
        )
    kmeans = KMeans(n_clusters=num_clusters, random_state=random_state).fit(
        embedded_documents
    )
    closest_indices = []
embedded_documents
    )
    closest_indices = []
    # Loop through the number of clusters you have
    for i in range(num_clusters):
        # Get the list of distances from that particular cluster center
        distances = np.linalg.norm(
            embedded_documents - kmeans.cluster_centers_[i], axis=1
        )
        # Find the indices of the two unique closest ones
        # (using argsort to find the smallest 2 distances)
        if remove_duplicates:
            # Only add not duplicated vectors.
            closest_indices_sorted = [
                x
                for x in np.argsort(distances)[:num_closest]
                if x not in closest_indices
            ]
        else:
            # Skip duplicates and add the next closest vector.
            closest_indices_sorted = [
                x for x in np.argsort(distances) if x not in closest_indices
            ][:num_closest]
        # Append that position closest indices list
        closest_indices.extend(closest_indices_sorted)
    return closest_indices
[docs]class EmbeddingsRedundantFilter(BaseDocumentTransformer, BaseModel):
    """Filter that drops redundant documents by comparing their embeddings."""
    embeddings: Embeddings
    """Embeddings to use for embedding document contents."""
    similarity_fn: Callable = cosine_similarity
    """Similarity function for comparing documents. Function expected to take as input
    two matrices (List[List[float]]) and return a matrix of scores where higher values
    indicate greater similarity."""
    similarity_threshold: float = 0.95
    """Threshold for determining when two documents are similar enough
    to be considered redundant."""
    class Config:
        """Configuration for this pydantic object."""
        arbitrary_types_allowed = True
[docs]    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Filter down documents."""
        stateful_documents = get_stateful_documents(documents)
        embedded_documents = _get_embeddings_from_stateful_docs(
            self.embeddings, stateful_documents
        )
        included_idxs = _filter_similar_embeddings(
            embedded_documents, self.similarity_fn, self.similarity_threshold
        )
        return [stateful_documents[i] for i in sorted(included_idxs)]
[docs]class EmbeddingsClusteringFilter(BaseDocumentTransformer, BaseModel):
    """Perform K-means clustering on document vectors.
    Returns an arbitrary number of documents closest to center."""
    embeddings: Embeddings
    """Embeddings to use for embedding document contents."""
    num_clusters: int = 5
    """Number of clusters. Groups of documents with similar meaning."""
    num_closest: int = 1
    """The number of closest vectors to return for each cluster center."""
    random_state: int = 42
    """Controls the random number generator used to initialize the cluster centroids.
    If you set the random_state parameter to None, the KMeans algorithm will use a 
    random number generator that is seeded with the current time. This means 
    that the results of the KMeans algorithm will be different each time you 
    run it."""
    sorted: bool = False
    """By default results are re-ordered "grouping" them by cluster, if sorted is true
    result will be ordered by the original position from the retriever"""
    remove_duplicates: bool = False
    """ By default duplicated results are skipped and replaced by the next closest 
    vector in the cluster. If remove_duplicates is true no replacement will be done:
vector in the cluster. If remove_duplicates is true no replacement will be done:
    This could dramatically reduce results when there is a lot of overlap between 
    clusters.
    """
    class Config:
        """Configuration for this pydantic object."""
        arbitrary_types_allowed = True
[docs]    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Filter down documents."""
        stateful_documents = get_stateful_documents(documents)
        embedded_documents = _get_embeddings_from_stateful_docs(
            self.embeddings, stateful_documents
        )
        included_idxs = _filter_cluster_embeddings(
            embedded_documents,
            self.num_clusters,
            self.num_closest,
            self.random_state,
            self.remove_duplicates,
        )
        results = sorted(included_idxs) if self.sorted else included_idxs
        return [stateful_documents[i] for i in results]
Source code for langchain_community.document_transformers.openai_functions
"""Document transformers that use OpenAI Functions models"""
from typing import Any, Dict, Optional, Sequence, Type, Union
from langchain_core.documents import BaseDocumentTransformer, Document
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.pydantic_v1 import BaseModel
[docs]class OpenAIMetadataTagger(BaseDocumentTransformer, BaseModel):
    """Extract metadata tags from document contents using OpenAI functions.
    Example:
        .. code-block:: python
                from langchain_community.chat_models import ChatOpenAI
                from langchain_community.document_transformers import OpenAIMetadataTagger
                from langchain_core.documents import Document
                schema = {
                    "properties": {
                        "movie_title": { "type": "string" },
                        "critic": { "type": "string" },
                        "tone": {
                            "type": "string",
                            "enum": ["positive", "negative"]
                        },
                        "rating": {
                            "type": "integer",
                            "description": "The number of stars the critic rated the movie"
                        }
                    },
                    "required": ["movie_title", "critic", "tone"]
                }
                # Must be an OpenAI model that supports functions
                llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613")
                tagging_chain = create_tagging_chain(schema, llm)
                document_transformer = OpenAIMetadataTagger(tagging_chain=tagging_chain)
                original_documents = [
                    Document(page_content="Review of The Bee Movie\nBy Roger Ebert\n\nThis is the greatest movie ever made. 4 out of 5 stars."),
Document(page_content="Review of The Godfather\nBy Anonymous\n\nThis movie was super boring. 1 out of 5 stars.", metadata={"reliable": False}),
                ]
                enhanced_documents = document_transformer.transform_documents(original_documents)
    """  # noqa: E501
    tagging_chain: Any
    """The chain used to extract metadata from each document."""
[docs]    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Automatically extract and populate metadata
        for each document according to the provided schema."""
        new_documents = []
        for document in documents:
            extracted_metadata: Dict = self.tagging_chain.run(document.page_content)  # type: ignore[assignment]  # noqa: E501
            new_document = Document(
                page_content=document.page_content,
                metadata={**extracted_metadata, **document.metadata},
            )
            new_documents.append(new_document)
        return new_documents
[docs]    async def atransform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        raise NotImplementedError
[docs]def create_metadata_tagger(
    metadata_schema: Union[Dict[str, Any], Type[BaseModel]],
    llm: BaseLanguageModel,
    prompt: Optional[ChatPromptTemplate] = None,
    *,
    tagging_chain_kwargs: Optional[Dict] = None,
) -> OpenAIMetadataTagger:
    """Create a DocumentTransformer that uses an OpenAI function chain to automatically
        tag documents with metadata based on their content and an input schema.
    Args:
        metadata_schema: Either a dictionary or pydantic.BaseModel class. If a dictionary
metadata_schema: Either a dictionary or pydantic.BaseModel class. If a dictionary
            is passed in, it's assumed to already be a valid JsonSchema.
            For best results, pydantic.BaseModels should have docstrings describing what
            the schema represents and descriptions for the parameters.
        llm: Language model to use, assumed to support the OpenAI function-calling API.
            Defaults to use "gpt-3.5-turbo-0613"
        prompt: BasePromptTemplate to pass to the model.
    Returns:
        An LLMChain that will pass the given function to the model.
    Example:
        .. code-block:: python
                from langchain_community.chat_models import ChatOpenAI
                from langchain_community.document_transformers import create_metadata_tagger
                from langchain_core.documents import Document
                schema = {
                    "properties": {
                        "movie_title": { "type": "string" },
                        "critic": { "type": "string" },
                        "tone": {
                            "type": "string",
                            "enum": ["positive", "negative"]
                        },
                        "rating": {
                            "type": "integer",
                            "description": "The number of stars the critic rated the movie"
                        }
                    },
                    "required": ["movie_title", "critic", "tone"]
                }
                # Must be an OpenAI model that supports functions
                llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613")
                document_transformer = create_metadata_tagger(schema, llm)
                original_documents = [
                    Document(page_content="Review of The Bee Movie\nBy Roger Ebert\n\nThis is the greatest movie ever made. 4 out of 5 stars."),
Document(page_content="Review of The Godfather\nBy Anonymous\n\nThis movie was super boring. 1 out of 5 stars.", metadata={"reliable": False}),
                ]
                enhanced_documents = document_transformer.transform_documents(original_documents)
    """  # noqa: E501
    from langchain.chains.openai_functions import create_tagging_chain
    metadata_schema = (
        metadata_schema
        if isinstance(metadata_schema, dict)
        else metadata_schema.schema()
    )
    _tagging_chain_kwargs = tagging_chain_kwargs or {}
    tagging_chain = create_tagging_chain(
        metadata_schema, llm, prompt=prompt, **_tagging_chain_kwargs
    )
    return OpenAIMetadataTagger(tagging_chain=tagging_chain)
Source code for langchain_community.document_transformers.google_translate
from typing import Any, Optional, Sequence
from langchain_core.documents import BaseDocumentTransformer, Document
from langchain_community.utilities.vertexai import get_client_info
[docs]class GoogleTranslateTransformer(BaseDocumentTransformer):
    """Translate text documents using Google Cloud Translation."""
[docs]    def __init__(
        self,
        project_id: str,
        *,
        location: str = "global",
        model_id: Optional[str] = None,
        glossary_id: Optional[str] = None,
        api_endpoint: Optional[str] = None,
    ) -> None:
        """
        Arguments:
            project_id: Google Cloud Project ID.
            location: (Optional) Translate model location.
            model_id: (Optional) Translate model ID to use.
            glossary_id: (Optional) Translate glossary ID to use.
            api_endpoint: (Optional) Regional endpoint to use.
        """
        try:
            from google.api_core.client_options import ClientOptions
            from google.cloud import translate
        except ImportError as exc:
            raise ImportError(
                "Install Google Cloud Translate to use this parser."
                "(pip install google-cloud-translate)"
            ) from exc
        self.project_id = project_id
        self.location = location
        self.model_id = model_id
        self.glossary_id = glossary_id
        self._client = translate.TranslationServiceClient(
            client_info=get_client_info("translate"),
            client_options=(
                ClientOptions(api_endpoint=api_endpoint) if api_endpoint else None
            ),
        )
        self._parent_path = self._client.common_location_path(project_id, location)
        # For some reason, there's no `model_path()` method for the client.
        self._model_path = (
self._model_path = (
            f"{self._parent_path}/models/{model_id}" if model_id else None
        )
        self._glossary_path = (
            self._client.glossary_path(project_id, location, glossary_id)
            if glossary_id
            else None
        )
[docs]    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        """Translate text documents using Google Translate.
        Arguments:
            source_language_code: ISO 639 language code of the input document.
            target_language_code: ISO 639 language code of the output document.
                For supported languages, refer to:
                https://cloud.google.com/translate/docs/languages
            mime_type: (Optional) Media Type of input text.
                Options: `text/plain`, `text/html`
        """
        try:
            from google.cloud import translate
        except ImportError as exc:
            raise ImportError(
                "Install Google Cloud Translate to use this parser."
                "(pip install google-cloud-translate)"
            ) from exc
        response = self._client.translate_text(
            request=translate.TranslateTextRequest(
                contents=[doc.page_content for doc in documents],
                parent=self._parent_path,
                model=self._model_path,
                glossary_config=translate.TranslateTextGlossaryConfig(
                    glossary=self._glossary_path
                ),
                source_language_code=kwargs.get("source_language_code", None),
                target_language_code=kwargs.get("target_language_code"),
                mime_type=kwargs.get("mime_type", "text/plain"),
            )
        )
        # If using a glossary, the translations will be in `glossary_translations`.
Source code for langchain_community.document_transformers.nuclia_text_transform
import asyncio
import json
import uuid
from typing import Any, Sequence
from langchain_core.documents import BaseDocumentTransformer, Document
from langchain_community.tools.nuclia.tool import NucliaUnderstandingAPI
[docs]class NucliaTextTransformer(BaseDocumentTransformer):
    """
    The Nuclia Understanding API splits into paragraphs and sentences,
    identifies entities, provides a summary of the text and generates
    embeddings for all sentences.
    """
[docs]    def __init__(self, nua: NucliaUnderstandingAPI):
        self.nua = nua
[docs]    def transform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        raise NotImplementedError
[docs]    async def atransform_documents(
        self, documents: Sequence[Document], **kwargs: Any
    ) -> Sequence[Document]:
        tasks = [
            self.nua.arun(
                {
                    "action": "push",
                    "id": str(uuid.uuid4()),
                    "text": doc.page_content,
                    "path": None,
                }
            )
            for doc in documents
        ]
        results = await asyncio.gather(*tasks)
        for doc, result in zip(documents, results):
            obj = json.loads(result)
            metadata = {
                "file": obj["file_extracted_data"][0],
                "metadata": obj["field_metadata"][0],
            }
            doc.metadata["nuclia"] = metadata
        return documents