viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.document_loaders.obs_directory
# coding:utf-8
from typing import List, Optional
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.obs_file import OBSFileLoader
[docs]class OBSDirectoryLoader(BaseLoader):
    """Load from `Huawei OBS directory`."""
[docs]    def __init__(
        self,
        bucket: str,
        endpoint: str,
        config: Optional[dict] = None,
        prefix: str = "",
    ):
        """Initialize the OBSDirectoryLoader with the specified settings.
        Args:
            bucket (str): The name of the OBS bucket to be used.
            endpoint (str): The endpoint URL of your OBS bucket.
            config (dict): The parameters for connecting to OBS, provided as a dictionary. The dictionary could have the following keys:
                - "ak" (str, optional): Your OBS access key (required if `get_token_from_ecs` is False and bucket policy is not public read).
                - "sk" (str, optional): Your OBS secret key (required if `get_token_from_ecs` is False and bucket policy is not public read).
                - "token" (str, optional): Your security token (required if using temporary credentials).
                - "get_token_from_ecs" (bool, optional): Whether to retrieve the security token from ECS. Defaults to False if not provided. If set to True, `ak`, `sk`, and `token` will be ignored.
            prefix (str, optional): The prefix to be added to the OBS key. Defaults to "".
        Note:
Note:
            Before using this class, make sure you have registered with OBS and have the necessary credentials. The `ak`, `sk`, and `endpoint` values are mandatory unless `get_token_from_ecs` is True or the bucket policy is public read. `token` is required when using temporary credentials.
        Example:
            To create a new OBSDirectoryLoader:
            ```
            config = {
                "ak": "your-access-key",
                "sk": "your-secret-key"
            }
            ```
            directory_loader = OBSDirectoryLoader("your-bucket-name", "your-end-endpoint", config, "your-prefix")
        """  # noqa: E501
        try:
            from obs import ObsClient
        except ImportError:
            raise ImportError(
                "Could not import esdk-obs-python python package. "
                "Please install it with `pip install esdk-obs-python`."
            )
        if not config:
            config = dict()
        if config.get("get_token_from_ecs"):
            self.client = ObsClient(server=endpoint, security_provider_policy="ECS")
        else:
            self.client = ObsClient(
                access_key_id=config.get("ak"),
                secret_access_key=config.get("sk"),
                security_token=config.get("token"),
                server=endpoint,
            )
        self.bucket = bucket
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        max_num = 1000
        mark = None
        docs = []
        while True:
            resp = self.client.listObjects(
                self.bucket, prefix=self.prefix, marker=mark, max_keys=max_num
            )
            if resp.status < 300:
Source code for langchain_community.document_loaders.acreom
import re
from pathlib import Path
from typing import Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class AcreomLoader(BaseLoader):
    """Load `acreom` vault from a directory."""
    FRONT_MATTER_REGEX = re.compile(r"^---\n(.*?)\n---\n", re.MULTILINE | re.DOTALL)
    """Regex to match front matter metadata in markdown files."""
[docs]    def __init__(
        self, path: str, encoding: str = "UTF-8", collect_metadata: bool = True
    ):
        """Initialize the loader."""
        self.file_path = path
        """Path to the directory containing the markdown files."""
        self.encoding = encoding
        """Encoding to use when reading the files."""
        self.collect_metadata = collect_metadata
        """Whether to collect metadata from the front matter."""
    def _parse_front_matter(self, content: str) -> dict:
        """Parse front matter metadata from the content and return it as a dict."""
        if not self.collect_metadata:
            return {}
        match = self.FRONT_MATTER_REGEX.search(content)
        front_matter = {}
        if match:
            lines = match.group(1).split("\n")
            for line in lines:
                if ":" in line:
                    key, value = line.split(":", 1)
                    front_matter[key.strip()] = value.strip()
                else:
                    # Skip lines without a colon
                    continue
        return front_matter
    def _remove_front_matter(self, content: str) -> str:
        """Remove front matter metadata from the given content."""
        if not self.collect_metadata:
            return content
if not self.collect_metadata:
            return content
        return self.FRONT_MATTER_REGEX.sub("", content)
    def _process_acreom_content(self, content: str) -> str:
        # remove acreom specific elements from content that
        # do not contribute to the context of current document
        content = re.sub(r"\s*-\s\[\s\]\s.*|\s*\[\s\]\s.*", "", content)  # rm tasks
        content = re.sub(r"#", "", content)  # rm hashtags
        content = re.sub(r"\[\[.*?\]\]", "", content)  # rm doclinks
        return content
[docs]    def lazy_load(self) -> Iterator[Document]:
        ps = list(Path(self.file_path).glob("**/*.md"))
        for p in ps:
            with open(p, encoding=self.encoding) as f:
                text = f.read()
            front_matter = self._parse_front_matter(text)
            text = self._remove_front_matter(text)
            text = self._process_acreom_content(text)
            metadata = {
                "source": str(p.name),
                "path": str(p),
                **front_matter,
            }
            yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.discord
from __future__ import annotations
from typing import TYPE_CHECKING, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import pandas as pd
[docs]class DiscordChatLoader(BaseLoader):
    """Load `Discord` chat logs."""
[docs]    def __init__(self, chat_log: pd.DataFrame, user_id_col: str = "ID"):
        """Initialize with a Pandas DataFrame containing chat logs.
        Args:
            chat_log: Pandas DataFrame containing chat logs.
            user_id_col: Name of the column containing the user ID. Defaults to "ID".
        """
        if not isinstance(chat_log, pd.DataFrame):
            raise ValueError(
                f"Expected chat_log to be a pd.DataFrame, got {type(chat_log)}"
            )
        self.chat_log = chat_log
        self.user_id_col = user_id_col
[docs]    def load(self) -> List[Document]:
        """Load all chat messages."""
        result = []
        for _, row in self.chat_log.iterrows():
            user_id = row[self.user_id_col]
            metadata = row.to_dict()
            metadata.pop(self.user_id_col)
            result.append(Document(page_content=user_id, metadata=metadata))
        return result
Source code for langchain_community.document_loaders.readthedocs
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Sequence, Tuple, Union
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    from bs4 import NavigableString
    from bs4.element import Comment, Tag
[docs]class ReadTheDocsLoader(BaseLoader):
    """Load `ReadTheDocs` documentation directory."""
[docs]    def __init__(
        self,
        path: Union[str, Path],
        encoding: Optional[str] = None,
        errors: Optional[str] = None,
        custom_html_tag: Optional[Tuple[str, dict]] = None,
        patterns: Sequence[str] = ("*.htm", "*.html"),
        exclude_links_ratio: float = 1.0,
        **kwargs: Optional[Any],
    ):
        """
        Initialize ReadTheDocsLoader
        The loader loops over all files under `path` and extracts the actual content of
        the files by retrieving main html tags. Default main html tags include
        `<main id="main-content>`, <`div role="main>`, and `<article role="main">`. You
        can also define your own html tags by passing custom_html_tag, e.g.
        `("div", "class=main")`. The loader iterates html tags with the order of
        custom html tags (if exists) and default html tags. If any of the tags is not
        empty, the loop will break and retrieve the content out of that tag.
        Args:
            path: The location of pulled readthedocs folder.
            encoding: The encoding with which to open the documents.
encoding: The encoding with which to open the documents.
            errors: Specify how encoding and decoding errors are to be handled—this
                cannot be used in binary mode.
            custom_html_tag: Optional custom html tag to retrieve the content from
                files.
            patterns: The file patterns to load, passed to `glob.rglob`.
            exclude_links_ratio: The ratio of links:content to exclude pages from.
                This is to reduce the frequency at which index pages make their
                way into retrieved results. Recommended: 0.5
            kwargs: named arguments passed to `bs4.BeautifulSoup`.
        """
        try:
            from bs4 import BeautifulSoup
        except ImportError:
            raise ImportError(
                "Could not import python packages. "
                "Please install it with `pip install beautifulsoup4`. "
            )
        try:
            _ = BeautifulSoup(
                "<html><body>Parser builder library test.</body></html>",
                "html.parser",
                **kwargs,
            )
        except Exception as e:
            raise ValueError("Parsing kwargs do not appear valid") from e
        self.file_path = Path(path)
        self.encoding = encoding
        self.errors = errors
        self.custom_html_tag = custom_html_tag
        self.patterns = patterns
        self.bs_kwargs = kwargs
        self.exclude_links_ratio = exclude_links_ratio
[docs]    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader for Documents."""
        for file_pattern in self.patterns:
            for p in self.file_path.rglob(file_pattern):
                if p.is_dir():
                    continue
                with open(p, encoding=self.encoding, errors=self.errors) as f:
                    text = self._clean_data(f.read())
text = self._clean_data(f.read())
                yield Document(page_content=text, metadata={"source": str(p)})
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        return list(self.lazy_load())
    def _clean_data(self, data: str) -> str:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(data, "html.parser", **self.bs_kwargs)
        # default tags
        html_tags = [
            ("div", {"role": "main"}),
            ("main", {"id": "main-content"}),
        ]
        if self.custom_html_tag is not None:
            html_tags.append(self.custom_html_tag)
        element = None
        # reversed order. check the custom one first
        for tag, attrs in html_tags[::-1]:
            element = soup.find(tag, attrs)
            # if found, break
            if element is not None:
                break
        if element is not None and _get_link_ratio(element) <= self.exclude_links_ratio:
            text = _get_clean_text(element)
        else:
            text = ""
        # trim empty lines
        return "\n".join([t for t in text.split("\n") if t])
def _get_clean_text(element: Tag) -> str:
    """Returns cleaned text with newlines preserved and irrelevant elements removed."""
    elements_to_skip = [
        "script",
        "noscript",
        "canvas",
        "meta",
        "svg",
        "map",
        "area",
        "audio",
        "source",
        "track",
        "video",
        "embed",
        "object",
        "param",
        "picture",
        "iframe",
        "frame",
        "frameset",
"iframe",
        "frame",
        "frameset",
        "noframes",
        "applet",
        "form",
        "button",
        "select",
        "base",
        "style",
        "img",
    ]
    newline_elements = [
        "p",
        "div",
        "ul",
        "ol",
        "li",
        "h1",
        "h2",
        "h3",
        "h4",
        "h5",
        "h6",
        "pre",
        "table",
        "tr",
    ]
    text = _process_element(element, elements_to_skip, newline_elements)
    return text.strip()
def _get_link_ratio(section: Tag) -> float:
    links = section.find_all("a")
    total_text = "".join(str(s) for s in section.stripped_strings)
    if len(total_text) == 0:
        return 0
    link_text = "".join(
        str(string.string.strip())
        for link in links
        for string in link.strings
        if string
    )
    return len(link_text) / len(total_text)
def _process_element(
    element: Union[Tag, NavigableString, Comment],
    elements_to_skip: List[str],
    newline_elements: List[str],
) -> str:
    """
    Traverse through HTML tree recursively to preserve newline and skip
    unwanted (code/binary) elements
    """
    from bs4 import NavigableString
    from bs4.element import Comment, Tag
    tag_name = getattr(element, "name", None)
    if isinstance(element, Comment) or tag_name in elements_to_skip:
        return ""
if isinstance(element, Comment) or tag_name in elements_to_skip:
        return ""
    elif isinstance(element, NavigableString):
        return element
    elif tag_name == "br":
        return "\n"
    elif tag_name in newline_elements:
        return (
            "".join(
                _process_element(child, elements_to_skip, newline_elements)
                for child in element.children
                if isinstance(child, (Tag, NavigableString, Comment))
            )
            + "\n"
        )
    else:
        return "".join(
            _process_element(child, elements_to_skip, newline_elements)
            for child in element.children
            if isinstance(child, (Tag, NavigableString, Comment))
        )
Source code for langchain_community.document_loaders.lakefs
import os
import tempfile
import urllib.parse
from typing import Any, List, Optional
from urllib.parse import urljoin
import requests
from langchain_core.documents import Document
from requests.auth import HTTPBasicAuth
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.unstructured import UnstructuredBaseLoader
[docs]class LakeFSClient:
[docs]    def __init__(
        self,
        lakefs_access_key: str,
        lakefs_secret_key: str,
        lakefs_endpoint: str,
    ):
        self.__endpoint = "/".join([lakefs_endpoint, "api", "v1/"])
        self.__auth = HTTPBasicAuth(lakefs_access_key, lakefs_secret_key)
        try:
            health_check = requests.get(
                urljoin(self.__endpoint, "healthcheck"), auth=self.__auth
            )
            health_check.raise_for_status()
        except Exception:
            raise ValueError(
                "lakeFS server isn't accessible. Make sure lakeFS is running."
            )
[docs]    def ls_objects(
        self, repo: str, ref: str, path: str, presign: Optional[bool]
    ) -> List:
        qp = {"prefix": path, "presign": presign}
        eqp = urllib.parse.urlencode(qp)
        objects_ls_endpoint = urljoin(
            self.__endpoint, f"repositories/{repo}/refs/{ref}/objects/ls?{eqp}"
        )
        olsr = requests.get(objects_ls_endpoint, auth=self.__auth)
        olsr.raise_for_status()
        olsr_json = olsr.json()
        return list(
            map(
return list(
            map(
                lambda res: (res["path"], res["physical_address"]), olsr_json["results"]
            )
        )
[docs]    def is_presign_supported(self) -> bool:
        config_endpoint = self.__endpoint + "config"
        response = requests.get(config_endpoint, auth=self.__auth)
        response.raise_for_status()
        config = response.json()
        return config["storage_config"]["pre_sign_support"]
[docs]class LakeFSLoader(BaseLoader):
    """Load from `lakeFS`."""
    repo: str
    ref: str
    path: str
[docs]    def __init__(
        self,
        lakefs_access_key: str,
        lakefs_secret_key: str,
        lakefs_endpoint: str,
        repo: Optional[str] = None,
        ref: Optional[str] = "main",
        path: Optional[str] = "",
    ):
        """
        :param lakefs_access_key: [required] lakeFS server's access key
        :param lakefs_secret_key: [required] lakeFS server's secret key
        :param lakefs_endpoint: [required] lakeFS server's endpoint address,
               ex: https://example.my-lakefs.com
        :param repo: [optional, default = ''] target repository
        :param ref: [optional, default = 'main'] target ref (branch name,
               tag, or commit ID)
        :param path: [optional, default = ''] target path
        """
        self.__lakefs_client = LakeFSClient(
            lakefs_access_key, lakefs_secret_key, lakefs_endpoint
        )
        self.repo = "" if repo is None or repo == "" else str(repo)
self.repo = "" if repo is None or repo == "" else str(repo)
        self.ref = "main" if ref is None or ref == "" else str(ref)
        self.path = "" if path is None else str(path)
[docs]    def set_path(self, path: str) -> None:
        self.path = path
[docs]    def set_ref(self, ref: str) -> None:
        self.ref = ref
[docs]    def set_repo(self, repo: str) -> None:
        self.repo = repo
[docs]    def load(self) -> List[Document]:
        self.__validate_instance()
        presigned = self.__lakefs_client.is_presign_supported()
        docs: List[Document] = []
        objs = self.__lakefs_client.ls_objects(
            repo=self.repo, ref=self.ref, path=self.path, presign=presigned
        )
        for obj in objs:
            lakefs_unstructured_loader = UnstructuredLakeFSLoader(
                obj[1], self.repo, self.ref, obj[0], presigned
            )
            docs.extend(lakefs_unstructured_loader.load())
        return docs
    def __validate_instance(self) -> None:
        if self.repo is None or self.repo == "":
            raise ValueError(
                "no repository was provided. use `set_repo` to specify a repository"
            )
        if self.ref is None or self.ref == "":
            raise ValueError("no ref was provided. use `set_ref` to specify a ref")
        if self.path is None:
            raise ValueError("no path was provided. use `set_path` to specify a path")
[docs]class UnstructuredLakeFSLoader(UnstructuredBaseLoader):
[docs]    def __init__(
        self,
[docs]    def __init__(
        self,
        url: str,
        repo: str,
        ref: str = "main",
        path: str = "",
        presign: bool = True,
        **unstructured_kwargs: Any,
    ):
        """
        Args:
        :param lakefs_access_key:
        :param lakefs_secret_key:
        :param lakefs_endpoint:
        :param repo:
        :param ref:
        """
        super().__init__(**unstructured_kwargs)
        self.url = url
        self.repo = repo
        self.ref = ref
        self.path = path
        self.presign = presign
    def _get_metadata(self) -> dict:
        return {"repo": self.repo, "ref": self.ref, "path": self.path}
    def _get_elements(self) -> List:
        from unstructured.partition.auto import partition
        local_prefix = "local://"
        if self.presign:
            with tempfile.TemporaryDirectory() as temp_dir:
                file_path = f"{temp_dir}/{self.path.split('/')[-1]}"
                os.makedirs(os.path.dirname(file_path), exist_ok=True)
                response = requests.get(self.url)
                response.raise_for_status()
                with open(file_path, mode="wb") as file:
                    file.write(response.content)
                return partition(filename=file_path)
        elif not self.url.startswith(local_prefix):
            raise ValueError(
                "Non pre-signed URLs are supported only with 'local' blockstore"
            )
        else:
            local_path = self.url[len(local_prefix) :]
            return partition(filename=local_path)
Source code for langchain_community.document_loaders.docugami
import hashlib
import io
import logging
import os
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Sequence, Union
import requests
from langchain_core.documents import Document
from langchain_core.pydantic_v1 import BaseModel, root_validator
from langchain_community.document_loaders.base import BaseLoader
TABLE_NAME = "{http://www.w3.org/1999/xhtml}table"
XPATH_KEY = "xpath"
ID_KEY = "id"
DOCUMENT_SOURCE_KEY = "source"
DOCUMENT_NAME_KEY = "name"
STRUCTURE_KEY = "structure"
TAG_KEY = "tag"
PROJECTS_KEY = "projects"
DEFAULT_API_ENDPOINT = "https://api.docugami.com/v1preview1"
logger = logging.getLogger(__name__)
[docs]class DocugamiLoader(BaseLoader, BaseModel):
    """Load from `Docugami`.
    To use, you should have the ``dgml-utils`` python package installed.
    """
    api: str = DEFAULT_API_ENDPOINT
    """The Docugami API endpoint to use."""
    access_token: Optional[str] = os.environ.get("DOCUGAMI_API_KEY")
    """The Docugami API access token to use."""
    max_text_length = 4096
    """Max length of chunk text returned."""
    min_text_length: int = 32
    """Threshold under which chunks are appended to next to avoid over-chunking."""
    max_metadata_length = 512
    """Max length of metadata text returned."""
    include_xml_tags: bool = False
    """Set to true for XML tags in chunk output text."""
    parent_hierarchy_levels: int = 0
    """Set appropriately to get parent chunks using the chunk hierarchy."""
"""Set appropriately to get parent chunks using the chunk hierarchy."""
    parent_id_key: str = "doc_id"
    """Metadata key for parent doc ID."""
    sub_chunk_tables: bool = False
    """Set to True to return sub-chunks within tables."""
    whitespace_normalize_text: bool = True
    """Set to False if you want to full whitespace formatting in the original
    XML doc, including indentation."""
    docset_id: Optional[str]
    """The Docugami API docset ID to use."""
    document_ids: Optional[Sequence[str]]
    """The Docugami API document IDs to use."""
    file_paths: Optional[Sequence[Union[Path, str]]]
    """The local file paths to use."""
    include_project_metadata_in_doc_metadata: bool = True
    """Set to True if you want to include the project metadata in the doc metadata."""
    @root_validator
    def validate_local_or_remote(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that either local file paths are given, or remote API docset ID.
        Args:
            values: The values to validate.
        Returns:
            The validated values.
        """
        if values.get("file_paths") and values.get("docset_id"):
            raise ValueError("Cannot specify both file_paths and remote API docset_id")
        if not values.get("file_paths") and not values.get("docset_id"):
            raise ValueError("Must specify either file_paths or remote API docset_id")
        if values.get("docset_id") and not values.get("access_token"):
            raise ValueError("Must specify access token if using remote API docset_id")
        return values
    def _parse_dgml(
        self,
        content: bytes,