viruPVS/langchain-docs huggingface dataset:
*tasks, desc="Fetching pages", ascii=True, mininterval=1
            )
        except ImportError:
            warnings.warn("For better logging of progress, `pip install tqdm`")
            return await asyncio.gather(*tasks)
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load text from the url(s) in web_path."""
        for doc in self.load():
            yield doc
[docs]    def load(self) -> List[Document]:
        """Load text from the url(s) in web_path."""
        try:
            # Raises RuntimeError if there is no current event loop.
            asyncio.get_running_loop()
            # If there is a current event loop, we need to run the async code
            # in a separate loop, in a separate thread.
            with ThreadPoolExecutor(max_workers=1) as executor:
                future = executor.submit(asyncio.run, self.fetch_all(self.web_paths))
                results = future.result()
        except RuntimeError:
            results = asyncio.run(self.fetch_all(self.web_paths))
        docs = []
        for i, text in enumerate(cast(List[str], results)):
            soup = self._scrape(self.web_paths[i])
            if not soup:
                continue
            metadata = _build_metadata(soup, self.web_paths[i])
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
Source code for langchain_community.document_loaders.excel
"""Loads Microsoft Excel files."""
from typing import Any, List
from langchain_community.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredExcelLoader(UnstructuredFileLoader):
    """Load Microsoft Excel files using `Unstructured`.
    Like other
    Unstructured loaders, UnstructuredExcelLoader can be used in both
    "single" and "elements" mode. If you use the loader in "elements"
    mode, each sheet in the Excel file will be a an Unstructured Table
    element. If you use the loader in "elements" mode, an
    HTML representation of the table will be available in the
    "text_as_html" key in the document metadata.
    Examples
    --------
    from langchain_community.document_loaders.excel import UnstructuredExcelLoader
    loader = UnstructuredExcelLoader("stanley-cups.xlsd", mode="elements")
    docs = loader.load()
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """
        Args:
            file_path: The path to the Microsoft Excel file.
            mode: The mode to use when partitioning the file. See unstructured docs
              for more info. Optional. Defaults to "single".
            **unstructured_kwargs: Keyword arguments to pass to unstructured.
        """
        validate_unstructured_version(min_unstructured_version="0.6.7")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.xlsx import partition_xlsx
Source code for langchain_community.document_loaders.fauna
from typing import Iterator, List, Optional, Sequence
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class FaunaLoader(BaseLoader):
    """Load from `FaunaDB`.
    Attributes:
        query (str): The FQL query string to execute.
        page_content_field (str): The field that contains the content of each page.
        secret (str): The secret key for authenticating to FaunaDB.
        metadata_fields (Optional[Sequence[str]]):
            Optional list of field names to include in metadata.
    """
[docs]    def __init__(
        self,
        query: str,
        page_content_field: str,
        secret: str,
        metadata_fields: Optional[Sequence[str]] = None,
    ):
        self.query = query
        self.page_content_field = page_content_field
        self.secret = secret
        self.metadata_fields = metadata_fields
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        try:
            from fauna import Page, fql
            from fauna.client import Client
            from fauna.encoding import QuerySuccess
        except ImportError:
            raise ImportError(
                "Could not import fauna python package. "
                "Please install it with `pip install fauna`."
            )
        # Create Fauna Client
        client = Client(secret=self.secret)
        # Run FQL Query
        response: QuerySuccess = client.query(fql(self.query))
        page: Page = response.data
        for result in page:
            if result is not None:
                document_dict = dict(result.items())
                page_content = ""
document_dict = dict(result.items())
                page_content = ""
                for key, value in document_dict.items():
                    if key == self.page_content_field:
                        page_content = value
                document: Document = Document(
                    page_content=page_content,
                    metadata={"id": result.id, "ts": result.ts},
                )
                yield document
        if page.after is not None:
            yield Document(
                page_content="Next Page Exists",
                metadata={"after": page.after},
            )
Source code for langchain_community.document_loaders.markdown
from typing import List
from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredMarkdownLoader(UnstructuredFileLoader):
    """Load `Markdown` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain_community.document_loaders import UnstructuredMarkdownLoader
    loader = UnstructuredMarkdownLoader(
        "example.md", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-md
    """
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.partition.md import partition_md
        # NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
        # versions of unstructured like 0.4.17-dev1
        _unstructured_version = __unstructured_version__.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        if unstructured_version < (0, 4, 16):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "
Source code for langchain_community.document_loaders.airbyte_json
import json
from typing import List
from langchain_core.documents import Document
from langchain_core.utils import stringify_dict
from langchain_community.document_loaders.base import BaseLoader
[docs]class AirbyteJSONLoader(BaseLoader):
    """Load local `Airbyte` json files."""
[docs]    def __init__(self, file_path: str):
        """Initialize with a file path. This should start with '/tmp/airbyte_local/'."""
        self.file_path = file_path
        """Path to the directory containing the json files."""
[docs]    def load(self) -> List[Document]:
        text = ""
        for line in open(self.file_path, "r"):
            data = json.loads(line)["_airbyte_data"]
            text += stringify_dict(data)
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
Source code for langchain_community.document_loaders.confluence
import logging
from enum import Enum
from io import BytesIO
from typing import Any, Callable, Dict, List, Optional, Union
import requests
from langchain_core.documents import Document
from tenacity import (
    before_sleep_log,
    retry,
    stop_after_attempt,
    wait_exponential,
)
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class ContentFormat(str, Enum):
    """Enumerator of the content formats of Confluence page."""
    EDITOR = "body.editor"
    EXPORT_VIEW = "body.export_view"
    ANONYMOUS_EXPORT_VIEW = "body.anonymous_export_view"
    STORAGE = "body.storage"
    VIEW = "body.view"
    def get_content(self, page: dict) -> str:
        return page["body"][self.name.lower()]["value"]
[docs]class ConfluenceLoader(BaseLoader):
    """Load `Confluence` pages.
    Port of https://llamahub.ai/l/confluence
    This currently supports username/api_key, Oauth2 login or personal access token
    authentication.
    Specify a list page_ids and/or space_key to load in the corresponding pages into
    Document objects, if both are specified the union of both sets will be returned.
    You can also specify a boolean `include_attachments` to include attachments, this
    is set to False by default, if set to True all attachments will be downloaded and
    ConfluenceReader will extract the text from the attachments and add it to the
    Document object. Currently supported attachment types are: PDF, PNG, JPEG/JPG,
    SVG, Word and Excel.
    Confluence API supports difference format of page content. The storage format is the
Confluence API supports difference format of page content. The storage format is the
    raw XML representation for storage. The view format is the HTML representation for
    viewing with macros are rendered as though it is viewed by users. You can pass
    a enum `content_format` argument to `load()` to specify the content format, this is
    set to `ContentFormat.STORAGE` by default, the supported values are:
    `ContentFormat.EDITOR`, `ContentFormat.EXPORT_VIEW`,
    `ContentFormat.ANONYMOUS_EXPORT_VIEW`, `ContentFormat.STORAGE`,
    and `ContentFormat.VIEW`.
    Hint: space_key and page_id can both be found in the URL of a page in Confluence
    - https://yoursite.atlassian.com/wiki/spaces/<space_key>/pages/<page_id>
    Example:
        .. code-block:: python
            from langchain_community.document_loaders import ConfluenceLoader
            loader = ConfluenceLoader(
                url="https://yoursite.atlassian.com/wiki",
                username="me",
                api_key="12345"
            )
            documents = loader.load(space_key="SPACE",limit=50)
            # Server on perm
            loader = ConfluenceLoader(
                url="https://confluence.yoursite.com/",
                username="me",
                api_key="your_password",
                cloud=False
            )
            documents = loader.load(space_key="SPACE",limit=50)
    :param url: _description_
    :type url: str
    :param api_key: _description_, defaults to None
    :type api_key: str, optional
    :param username: _description_, defaults to None
    :type username: str, optional
    :param oauth2: _description_, defaults to {}
:param oauth2: _description_, defaults to {}
    :type oauth2: dict, optional
    :param token: _description_, defaults to None
    :type token: str, optional
    :param cloud: _description_, defaults to True
    :type cloud: bool, optional
    :param number_of_retries: How many times to retry, defaults to 3
    :type number_of_retries: Optional[int], optional
    :param min_retry_seconds: defaults to 2
    :type min_retry_seconds: Optional[int], optional
    :param max_retry_seconds:  defaults to 10
    :type max_retry_seconds: Optional[int], optional
    :param confluence_kwargs: additional kwargs to initialize confluence with
    :type confluence_kwargs: dict, optional
    :raises ValueError: Errors while validating input
    :raises ImportError: Required dependencies not installed.
    """
[docs]    def __init__(
        self,
        url: str,
        api_key: Optional[str] = None,
        username: Optional[str] = None,
        session: Optional[requests.Session] = None,
        oauth2: Optional[dict] = None,
        token: Optional[str] = None,
        cloud: Optional[bool] = True,
        number_of_retries: Optional[int] = 3,
        min_retry_seconds: Optional[int] = 2,
        max_retry_seconds: Optional[int] = 10,
        confluence_kwargs: Optional[dict] = None,
    ):
        confluence_kwargs = confluence_kwargs or {}
        errors = ConfluenceLoader.validate_init_args(
            url=url,
            api_key=api_key,
            username=username,
            session=session,
            oauth2=oauth2,
            token=token,
        )
oauth2=oauth2,
            token=token,
        )
        if errors:
            raise ValueError(f"Error(s) while validating input: {errors}")
        try:
            from atlassian import Confluence  # noqa: F401
        except ImportError:
            raise ImportError(
                "`atlassian` package not found, please run "
                "`pip install atlassian-python-api`"
            )
        self.base_url = url
        self.number_of_retries = number_of_retries
        self.min_retry_seconds = min_retry_seconds
        self.max_retry_seconds = max_retry_seconds
        if session:
            self.confluence = Confluence(url=url, session=session, **confluence_kwargs)
        elif oauth2:
            self.confluence = Confluence(
                url=url, oauth2=oauth2, cloud=cloud, **confluence_kwargs
            )
        elif token:
            self.confluence = Confluence(
                url=url, token=token, cloud=cloud, **confluence_kwargs
            )
        else:
            self.confluence = Confluence(
                url=url,
                username=username,
                password=api_key,
                cloud=cloud,
                **confluence_kwargs,
            )
[docs]    @staticmethod
    def validate_init_args(
        url: Optional[str] = None,
        api_key: Optional[str] = None,
        username: Optional[str] = None,
        session: Optional[requests.Session] = None,
        oauth2: Optional[dict] = None,
        token: Optional[str] = None,
    ) -> Union[List, None]:
        """Validates proper combinations of init arguments"""
        errors = []
        if url is None:
            errors.append("Must provide `base_url`")
if url is None:
            errors.append("Must provide `base_url`")
        if (api_key and not username) or (username and not api_key):
            errors.append(
                "If one of `api_key` or `username` is provided, "
                "the other must be as well."
            )
        non_null_creds = list(
            x is not None for x in ((api_key or username), session, oauth2, token)
        )
        if sum(non_null_creds) > 1:
            all_names = ("(api_key, username)", "session", "oath2", "token")
            provided = tuple(n for x, n in zip(non_null_creds, all_names) if x)
            errors.append(
                f"Cannot provide a value for more than one of: {all_names}. Received "
                f"values for: {provided}"
            )
        if oauth2 and set(oauth2.keys()) != {
            "access_token",
            "access_token_secret",
            "consumer_key",
            "key_cert",
        }:
            errors.append(
                "You have either omitted require keys or added extra "
                "keys to the oauth2 dictionary. key values should be "
                "`['access_token', 'access_token_secret', 'consumer_key', 'key_cert']`"
            )
        return errors or None
[docs]    def load(
        self,
        space_key: Optional[str] = None,
        page_ids: Optional[List[str]] = None,
        label: Optional[str] = None,
        cql: Optional[str] = None,
        include_restricted_content: bool = False,
        include_archived_content: bool = False,
        include_attachments: bool = False,
include_archived_content: bool = False,
        include_attachments: bool = False,
        include_comments: bool = False,
        content_format: ContentFormat = ContentFormat.STORAGE,
        limit: Optional[int] = 50,
        max_pages: Optional[int] = 1000,
        ocr_languages: Optional[str] = None,
        keep_markdown_format: bool = False,
        keep_newlines: bool = False,
    ) -> List[Document]:
        """
        :param space_key: Space key retrieved from a confluence URL, defaults to None
        :type space_key: Optional[str], optional
        :param page_ids: List of specific page IDs to load, defaults to None
        :type page_ids: Optional[List[str]], optional
        :param label: Get all pages with this label, defaults to None
        :type label: Optional[str], optional
        :param cql: CQL Expression, defaults to None
        :type cql: Optional[str], optional
        :param include_restricted_content: defaults to False
        :type include_restricted_content: bool, optional
        :param include_archived_content: Whether to include archived content,
                                         defaults to False
        :type include_archived_content: bool, optional
        :param include_attachments: defaults to False
        :type include_attachments: bool, optional
        :param include_comments: defaults to False
        :type include_comments: bool, optional
        :param content_format: Specify content format, defaults to
                                ContentFormat.STORAGE, the supported values are:
                                `ContentFormat.EDITOR`, `ContentFormat.EXPORT_VIEW`,
                                `ContentFormat.ANONYMOUS_EXPORT_VIEW`,
                                `ContentFormat.STORAGE`, and `ContentFormat.VIEW`.
        :type content_format: ContentFormat
:type content_format: ContentFormat
        :param limit: Maximum number of pages to retrieve per request, defaults to 50
        :type limit: int, optional
        :param max_pages: Maximum number of pages to retrieve in total, defaults 1000
        :type max_pages: int, optional
        :param ocr_languages: The languages to use for the Tesseract agent. To use a
                              language, you'll first need to install the appropriate
                              Tesseract language pack.
        :type ocr_languages: str, optional
        :param keep_markdown_format: Whether to keep the markdown format, defaults to
            False
        :type keep_markdown_format: bool
        :param keep_newlines: Whether to keep the newlines format, defaults to
            False
        :type keep_newlines: bool
        :raises ValueError: _description_
        :raises ImportError: _description_
        :return: _description_
        :rtype: List[Document]
        """
        if not space_key and not page_ids and not label and not cql:
            raise ValueError(
                "Must specify at least one among `space_key`, `page_ids`, "
                "`label`, `cql` parameters."
            )
        docs = []
        if space_key:
            pages = self.paginate_request(
                self.confluence.get_all_pages_from_space,
                space=space_key,
                limit=limit,
                max_pages=max_pages,
                status="any" if include_archived_content else "current",
                expand=content_format.value,
            )
            docs += self.process_pages(
                pages,
                include_restricted_content,
                include_attachments,
                include_comments,
                content_format,
                ocr_languages=ocr_languages,
include_comments,
                content_format,
                ocr_languages=ocr_languages,
                keep_markdown_format=keep_markdown_format,
                keep_newlines=keep_newlines,
            )
        if label:
            pages = self.paginate_request(
                self.confluence.get_all_pages_by_label,
                label=label,
                limit=limit,
                max_pages=max_pages,
            )
            ids_by_label = [page["id"] for page in pages]
            if page_ids:
                page_ids = list(set(page_ids + ids_by_label))
            else:
                page_ids = list(set(ids_by_label))
        if cql:
            pages = self.paginate_request(
                self._search_content_by_cql,
                cql=cql,
                limit=limit,
                max_pages=max_pages,
                include_archived_spaces=include_archived_content,
                expand=content_format.value,
            )
            docs += self.process_pages(
                pages,
                include_restricted_content,
                include_attachments,
                include_comments,
                content_format,
                ocr_languages,
                keep_markdown_format,
            )
        if page_ids:
            for page_id in page_ids:
                get_page = retry(
                    reraise=True,
                    stop=stop_after_attempt(
                        self.number_of_retries  # type: ignore[arg-type]
                    ),
                    wait=wait_exponential(
                        multiplier=1,  # type: ignore[arg-type]
                        min=self.min_retry_seconds,  # type: ignore[arg-type]
                        max=self.max_retry_seconds,  # type: ignore[arg-type]
                    ),
                    before_sleep=before_sleep_log(logger, logging.WARNING),
                )(self.confluence.get_page_by_id)
)(self.confluence.get_page_by_id)
                page = get_page(
                    page_id=page_id, expand=f"{content_format.value},version"
                )
                if not include_restricted_content and not self.is_public_page(page):
                    continue
                doc = self.process_page(
                    page,
                    include_attachments,
                    include_comments,
                    content_format,
                    ocr_languages,
                    keep_markdown_format,
                )
                docs.append(doc)
        return docs
    def _search_content_by_cql(
        self, cql: str, include_archived_spaces: Optional[bool] = None, **kwargs: Any
    ) -> List[dict]:
        url = "rest/api/content/search"
        params: Dict[str, Any] = {"cql": cql}
        params.update(kwargs)
        if include_archived_spaces is not None:
            params["includeArchivedSpaces"] = include_archived_spaces
        response = self.confluence.get(url, params=params)
        return response.get("results", [])
[docs]    def paginate_request(self, retrieval_method: Callable, **kwargs: Any) -> List:
        """Paginate the various methods to retrieve groups of pages.
        Unfortunately, due to page size, sometimes the Confluence API
        doesn't match the limit value. If `limit` is >100 confluence
        seems to cap the response to 100. Also, due to the Atlassian Python
        package, we don't get the "next" values from the "_links" key because
        they only return the value from the result key. So here, the pagination
        starts from 0 and goes until the max_pages, getting the `limit` number
        of pages with each request. We have to manually check if there