mime_types_mapping[
                    file_type.value
                ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"  # noqa: E501
            elif file_type.value == "pdf":
                mime_types_mapping[file_type.value] = "application/pdf"
        return mime_types_mapping
[docs]class OneDriveLoader(BaseLoader, BaseModel):
    settings: _OneDriveSettings = Field(default_factory=_OneDriveSettings)
    drive_id: str = Field(...)
    folder_path: Optional[str] = None
    object_ids: Optional[List[str]] = None
    auth_with_token: bool = False
    def _auth(self) -> Type[Account]:
        """
        Authenticates the OneDrive API client using the specified
        authentication method and returns the Account object.
        Returns:
            Type[Account]: The authenticated Account object.
        """
        try:
            from O365 import FileSystemTokenBackend
        except ImportError:
            raise ImportError(
                "O365 package not found, please install it with `pip install o365`"
            )
        if self.auth_with_token:
            token_storage = _OneDriveTokenStorage()
            token_path = token_storage.token_path
            token_backend = FileSystemTokenBackend(
                token_path=token_path.parent, token_filename=token_path.name
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=SCOPES,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
        else:
            token_backend = FileSystemTokenBackend(
                token_path=Path.home() / ".credentials"
            )
            account = Account(
                credentials=(
                    self.settings.client_id,


)
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=SCOPES,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
            # make the auth
            account.authenticate()
        return account
    def _get_folder_from_path(self, drive: Type[Drive]) -> Union[Folder, Drive]:
        """
        Returns the folder or drive object located at the
        specified path relative to the given drive.
        Args:
            drive (Type[Drive]): The root drive from which the folder path is relative.
        Returns:
            Union[Folder, Drive]: The folder or drive object
            located at the specified path.
        Raises:
            FileNotFoundError: If the path does not exist.
        """
        subfolder_drive = drive
        if self.folder_path is None:
            return subfolder_drive
        subfolders = [f for f in self.folder_path.split("/") if f != ""]
        if len(subfolders) == 0:
            return subfolder_drive
        items = subfolder_drive.get_items()
        for subfolder in subfolders:
            try:
                subfolder_drive = list(filter(lambda x: subfolder in x.name, items))[0]
                items = subfolder_drive.get_items()
            except (IndexError, AttributeError):
                raise FileNotFoundError("Path {} not exist.".format(self.folder_path))
        return subfolder_drive
    def _load_from_folder(self, folder: Type[Folder]) -> List[Document]:
        """
        Loads all supported document files from the specified folder
        and returns a list of Document objects.
        Args:
            folder (Type[Folder]): The folder object to load the documents from.


folder (Type[Folder]): The folder object to load the documents from.
        Returns:
            List[Document]: A list of Document objects representing
            the loaded documents.
        """
        docs = []
        file_types = _SupportedFileTypes(file_types=["doc", "docx", "pdf"])
        file_mime_types = file_types.fetch_mime_types()
        items = folder.get_items()
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            for file in items:
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        loader = OneDriveFileLoader(file=file)
                        docs.extend(loader.load())
        return docs
    def _load_from_object_ids(self, drive: Type[Drive]) -> List[Document]:
        """
        Loads all supported document files from the specified OneDrive
        drive based on their object IDs and returns a list
        of Document objects.
        Args:
            drive (Type[Drive]): The OneDrive drive object
            to load the documents from.
        Returns:
            List[Document]: A list of Document objects representing
            the loaded documents.
        """
        docs = []
        file_types = _SupportedFileTypes(file_types=["doc", "docx", "pdf"])
        file_mime_types = file_types.fetch_mime_types()
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            for object_id in self.object_ids if self.object_ids else [""]:
                file = drive.get_item(object_id)
                if not file:
                    logging.warning(
                        "There isn't a file with "


logging.warning(
                        "There isn't a file with "
                        f"object_id {object_id} in drive {drive}."
                    )
                    continue
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        loader = OneDriveFileLoader(file=file)
                        docs.extend(loader.load())
        return docs
[docs]    def load(self) -> List[Document]:
        """
        Loads all supported document files from the specified OneDrive drive a
        nd returns a list of Document objects.
        Returns:
            List[Document]: A list of Document objects
            representing the loaded documents.
        Raises:
            ValueError: If the specified drive ID
            does not correspond to a drive in the OneDrive storage.
        """
        account = self._auth()
        storage = account.storage()
        drive = storage.get_drive(self.drive_id)
        docs: List[Document] = []
        if not drive:
            raise ValueError(f"There isn't a drive with id {self.drive_id}.")
        if self.folder_path:
            folder = self._get_folder_from_path(drive=drive)
            docs.extend(self._load_from_folder(folder=folder))
        elif self.object_ids:
            docs.extend(self._load_from_object_ids(drive=drive))
        return docs


Source code for langchain.document_loaders.json_loader
"""Loads data from JSON."""
import json
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class JSONLoader(BaseLoader):
    """Loads a JSON file using a jq schema.
    Example:
        [{"text": ...}, {"text": ...}, {"text": ...}] -> schema = .[].text
        {"key": [{"text": ...}, {"text": ...}, {"text": ...}]} -> schema = .key[].text
        ["", "", ""] -> schema = .[]
    """
    def __init__(
        self,
        file_path: Union[str, Path],
        jq_schema: str,
        content_key: Optional[str] = None,
        metadata_func: Optional[Callable[[Dict, Dict], Dict]] = None,
        text_content: bool = True,
        json_lines: bool = False,
    ):
        """Initialize the JSONLoader.
        Args:
            file_path (Union[str, Path]): The path to the JSON or JSON Lines file.
            jq_schema (str): The jq schema to use to extract the data or text from
                the JSON.
            content_key (str): The key to use to extract the content from the JSON if
                the jq_schema results to a list of objects (dict).
            metadata_func (Callable[Dict, Dict]): A function that takes in the JSON
                object extracted by the jq_schema and the default metadata and returns
                a dict of the updated metadata.
            text_content (bool): Boolean flag to indicate whether the content is in
                string format, default to True.
            json_lines (bool): Boolean flag to indicate whether the input is in


json_lines (bool): Boolean flag to indicate whether the input is in
                JSON Lines format.
        """
        try:
            import jq  # noqa:F401
        except ImportError:
            raise ImportError(
                "jq package not found, please install it with `pip install jq`"
            )
        self.file_path = Path(file_path).resolve()
        self._jq_schema = jq.compile(jq_schema)
        self._content_key = content_key
        self._metadata_func = metadata_func
        self._text_content = text_content
        self._json_lines = json_lines
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the JSON file."""
        docs: List[Document] = []
        if self._json_lines:
            with self.file_path.open(encoding="utf-8") as f:
                for line in f:
                    line = line.strip()
                    if line:
                        self._parse(line, docs)
        else:
            self._parse(self.file_path.read_text(), docs)
        return docs
    def _parse(self, content: str, docs: List[Document]) -> None:
        """Convert given content to documents."""
        data = self._jq_schema.input(json.loads(content))
        # Perform some validation
        # This is not a perfect validation, but it should catch most cases
        # and prevent the user from getting a cryptic error later on.
        if self._content_key is not None:
            self._validate_content_key(data)
        for i, sample in enumerate(data, len(docs) + 1):
            metadata = dict(
                source=str(self.file_path),
                seq_num=i,
            )
            text = self._get_text(sample=sample, metadata=metadata)


)
            text = self._get_text(sample=sample, metadata=metadata)
            docs.append(Document(page_content=text, metadata=metadata))
    def _get_text(self, sample: Any, metadata: dict) -> str:
        """Convert sample to string format"""
        if self._content_key is not None:
            content = sample.get(self._content_key)
            if self._metadata_func is not None:
                # We pass in the metadata dict to the metadata_func
                # so that the user can customize the default metadata
                # based on the content of the JSON object.
                metadata = self._metadata_func(sample, metadata)
        else:
            content = sample
        if self._text_content and not isinstance(content, str):
            raise ValueError(
                f"Expected page_content is string, got {type(content)} instead. \
                    Set `text_content=False` if the desired input for \
                    `page_content` is not a string"
            )
        # In case the text is None, set it to an empty string
        elif isinstance(content, str):
            return content
        elif isinstance(content, dict):
            return json.dumps(content) if content else ""
        else:
            return str(content) if content is not None else ""
    def _validate_content_key(self, data: Any) -> None:
        """Check if a content key is valid"""
        sample = data.first()
        if not isinstance(sample, dict):
            raise ValueError(
                f"Expected the jq schema to result in a list of objects (dict), \
                    so sample must be a dict but got `{type(sample)}`"
            )
        if sample.get(self._content_key) is None:
            raise ValueError(


if sample.get(self._content_key) is None:
            raise ValueError(
                f"Expected the jq schema to result in a list of objects (dict) \
                    with the key `{self._content_key}`"
            )
        if self._metadata_func is not None:
            sample_metadata = self._metadata_func(sample, {})
            if not isinstance(sample_metadata, dict):
                raise ValueError(
                    f"Expected the metadata_func to return a dict but got \
                        `{type(sample_metadata)}`"
                )


Source code for langchain.document_loaders.figma
"""Loader that loads Figma files json dump."""
import json
import urllib.request
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import stringify_dict
[docs]class FigmaFileLoader(BaseLoader):
    """Loads Figma file json."""
    def __init__(self, access_token: str, ids: str, key: str):
        """Initialize with access token, ids, and key.
        Args:
            access_token: The access token for the Figma REST API.
            ids: The ids of the Figma file.
            key: The key for the Figma file
        """
        self.access_token = access_token
        self.ids = ids
        self.key = key
    def _construct_figma_api_url(self) -> str:
        api_url = "https://api.figma.com/v1/files/%s/nodes?ids=%s" % (
            self.key,
            self.ids,
        )
        return api_url
    def _get_figma_file(self) -> Any:
        """Get Figma file from Figma REST API."""
        headers = {"X-Figma-Token": self.access_token}
        request = urllib.request.Request(
            self._construct_figma_api_url(), headers=headers
        )
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            return json_data
[docs]    def load(self) -> List[Document]:
        """Load file"""
        data = self._get_figma_file()
        text = stringify_dict(data)
        metadata = {"source": self._construct_figma_api_url()}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.s3_file
"""Loading logic for loading documents from an s3 file."""
import os
import tempfile
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class S3FileLoader(BaseLoader):
    """Loading logic for loading documents from s3."""
    def __init__(self, bucket: str, key: str):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.key = key
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            import boto3
        except ImportError:
            raise ImportError(
                "Could not import `boto3` python package. "
                "Please install it with `pip install boto3`."
            )
        s3 = boto3.client("s3")
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.key}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            s3.download_file(self.bucket, self.key, file_path)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()


Source code for langchain.document_loaders.url_playwright
"""Loader that uses Playwright to load a page, then uses unstructured to load the html.
"""
import logging
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class PlaywrightURLLoader(BaseLoader):
    """Loader that uses Playwright and to load a page and unstructured to load the html.
    This is useful for loading pages that require javascript to render.
    Attributes:
        urls (List[str]): List of URLs to load.
        continue_on_failure (bool): If True, continue loading other URLs on failure.
        headless (bool): If True, the browser will run in headless mode.
    """
    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        headless: bool = True,
        remove_selectors: Optional[List[str]] = None,
    ):
        """Load a list of URLs using Playwright and unstructured."""
        try:
            import playwright  # noqa:F401
        except ImportError:
            raise ImportError(
                "playwright package not found, please install it with "
                "`pip install playwright`"
            )
        try:
            import unstructured  # noqa:F401
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.headless = headless
        self.remove_selectors = remove_selectors
[docs]    def load(self) -> List[Document]:


[docs]    def load(self) -> List[Document]:
        """Load the specified URLs using Playwright and create Document instances.
        Returns:
            List[Document]: A list of Document instances with loaded content.
        """
        from playwright.sync_api import sync_playwright
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        with sync_playwright() as p:
            browser = p.chromium.launch(headless=self.headless)
            for url in self.urls:
                try:
                    page = browser.new_page()
                    page.goto(url)
                    for selector in self.remove_selectors or []:
                        elements = page.locator(selector).all()
                        for element in elements:
                            if element.is_visible():
                                element.evaluate("element => element.remove()")
                    page_source = page.content()
                    elements = partition_html(text=page_source)
                    text = "\n\n".join([str(el) for el in elements])
                    metadata = {"source": url}
                    docs.append(Document(page_content=text, metadata=metadata))
                except Exception as e:
                    if self.continue_on_failure:
                        logger.error(
                            f"Error fetching or processing {url}, exception: {e}"
                        )
                    else:
                        raise e
            browser.close()
        return docs


Source code for langchain.document_loaders.discord
"""Load from Discord chat dump"""
from __future__ import annotations
from typing import TYPE_CHECKING, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import pandas as pd
[docs]class DiscordChatLoader(BaseLoader):
    """Load Discord chat logs."""
    def __init__(self, chat_log: pd.DataFrame, user_id_col: str = "ID"):
        """Initialize with a Pandas DataFrame containing chat logs.
        Args:
            chat_log: Pandas DataFrame containing chat logs.
            user_id_col: Name of the column containing the user ID. Defaults to "ID".
        """
        if not isinstance(chat_log, pd.DataFrame):
            raise ValueError(
                f"Expected chat_log to be a pd.DataFrame, got {type(chat_log)}"
            )
        self.chat_log = chat_log
        self.user_id_col = user_id_col
[docs]    def load(self) -> List[Document]:
        """Load all chat messages."""
        result = []
        for _, row in self.chat_log.iterrows():
            user_id = row[self.user_id_col]
            metadata = row.to_dict()
            metadata.pop(self.user_id_col)
            result.append(Document(page_content=user_id, metadata=metadata))
        return result


Source code for langchain.document_loaders.url_selenium
"""Loader that uses Selenium to load a page, then uses unstructured to load the html.
"""
import logging
from typing import TYPE_CHECKING, List, Literal, Optional, Union
if TYPE_CHECKING:
    from selenium.webdriver import Chrome, Firefox
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class SeleniumURLLoader(BaseLoader):
    """Loader that uses Selenium and to load a page and unstructured to load the html.
    This is useful for loading pages that require javascript to render.
    Attributes:
        urls (List[str]): List of URLs to load.
        continue_on_failure (bool): If True, continue loading other URLs on failure.
        browser (str): The browser to use, either 'chrome' or 'firefox'.
        binary_location (Optional[str]): The location of the browser binary.
        executable_path (Optional[str]): The path to the browser executable.
        headless (bool): If True, the browser will run in headless mode.
        arguments [List[str]]: List of arguments to pass to the browser.
    """
    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        browser: Literal["chrome", "firefox"] = "chrome",
        binary_location: Optional[str] = None,
        executable_path: Optional[str] = None,
        headless: bool = True,
        arguments: List[str] = [],
    ):
        """Load a list of URLs using Selenium and unstructured."""
        try:
            import selenium  # noqa:F401
        except ImportError:
            raise ImportError(
                "selenium package not found, please install it with "


raise ImportError(
                "selenium package not found, please install it with "
                "`pip install selenium`"
            )
        try:
            import unstructured  # noqa:F401
        except ImportError:
            raise ImportError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.browser = browser
        self.binary_location = binary_location
        self.executable_path = executable_path
        self.headless = headless
        self.arguments = arguments
    def _get_driver(self) -> Union["Chrome", "Firefox"]:
        """Create and return a WebDriver instance based on the specified browser.
        Raises:
            ValueError: If an invalid browser is specified.
        Returns:
            Union[Chrome, Firefox]: A WebDriver instance for the specified browser.
        """
        if self.browser.lower() == "chrome":
            from selenium.webdriver import Chrome
            from selenium.webdriver.chrome.options import Options as ChromeOptions
            chrome_options = ChromeOptions()
            for arg in self.arguments:
                chrome_options.add_argument(arg)
            if self.headless:
                chrome_options.add_argument("--headless")
                chrome_options.add_argument("--no-sandbox")
            if self.binary_location is not None:
                chrome_options.binary_location = self.binary_location
            if self.executable_path is None:
                return Chrome(options=chrome_options)
            return Chrome(executable_path=self.executable_path, options=chrome_options)
        elif self.browser.lower() == "firefox":
            from selenium.webdriver import Firefox
            from selenium.webdriver.firefox.options import Options as FirefoxOptions
            firefox_options = FirefoxOptions()
            for arg in self.arguments:
                firefox_options.add_argument(arg)


for arg in self.arguments:
                firefox_options.add_argument(arg)
            if self.headless:
                firefox_options.add_argument("--headless")
            if self.binary_location is not None:
                firefox_options.binary_location = self.binary_location
            if self.executable_path is None:
                return Firefox(options=firefox_options)
            return Firefox(
                executable_path=self.executable_path, options=firefox_options
            )
        else:
            raise ValueError("Invalid browser specified. Use 'chrome' or 'firefox'.")
[docs]    def load(self) -> List[Document]:
        """Load the specified URLs using Selenium and create Document instances.
        Returns:
            List[Document]: A list of Document instances with loaded content.
        """
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        driver = self._get_driver()
        for url in self.urls:
            try:
                driver.get(url)
                page_content = driver.page_source
                elements = partition_html(text=page_content)
                text = "\n\n".join([str(el) for el in elements])
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                else:
                    raise e
        driver.quit()
        return docs


Source code for langchain.document_loaders.open_city_data
from typing import Iterator, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class OpenCityDataLoader(BaseLoader):
    """Loader that loads Open city data."""
    def __init__(self, city_id: str, dataset_id: str, limit: int):
        """Initialize with dataset_id"""
        """ Example: https://dev.socrata.com/foundry/data.sfgov.org/vw6y-z8j6 """
        """ e.g., city_id = data.sfgov.org """
        """ e.g., dataset_id = vw6y-z8j6 """
        self.city_id = city_id
        self.dataset_id = dataset_id
        self.limit = limit
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load records."""
        from sodapy import Socrata
        client = Socrata(self.city_id, None)
        results = client.get(self.dataset_id, limit=self.limit)
        for record in results:
            yield Document(
                page_content=str(record),
                metadata={
                    "source": self.city_id + "_" + self.dataset_id,
                },
            )
[docs]    def load(self) -> List[Document]:
        """Load records."""
        return list(self.lazy_load())


Source code for langchain.document_loaders.sitemap
"""Loader that fetches a sitemap and loads those URLs."""
import itertools
import re
from typing import Any, Callable, Generator, Iterable, List, Optional
from langchain.document_loaders.web_base import WebBaseLoader
from langchain.schema import Document
def _default_parsing_function(content: Any) -> str:
    return str(content.get_text())
def _default_meta_function(meta: dict, _content: Any) -> dict:
    return {"source": meta["loc"], **meta}
def _batch_block(iterable: Iterable, size: int) -> Generator[List[dict], None, None]:
    it = iter(iterable)
    while item := list(itertools.islice(it, size)):
        yield item
[docs]class SitemapLoader(WebBaseLoader):
    """Loader that fetches a sitemap and loads those URLs."""
    def __init__(
        self,
        web_path: str,
        filter_urls: Optional[List[str]] = None,
        parsing_function: Optional[Callable] = None,
        blocksize: Optional[int] = None,
        blocknum: int = 0,
        meta_function: Optional[Callable] = None,
        is_local: bool = False,
    ):
        """Initialize with webpage path and optional filter URLs.
        Args:
            web_path: url of the sitemap. can also be a local path
            filter_urls: list of strings or regexes that will be applied to filter the
                urls that are parsed and loaded
            parsing_function: Function to parse bs4.Soup output
            blocksize: number of sitemap locations per block
            blocknum: the number of the block that should be loaded - zero indexed
            meta_function: Function to parse bs4.Soup output for metadata


meta_function: Function to parse bs4.Soup output for metadata
                remember when setting this method to also copy metadata["loc"]
                to metadata["source"] if you are using this field
            is_local: whether the sitemap is a local file
        """
        if blocksize is not None and blocksize < 1:
            raise ValueError("Sitemap blocksize should be at least 1")
        if blocknum < 0:
            raise ValueError("Sitemap blocknum can not be lower then 0")
        try:
            import lxml  # noqa:F401
        except ImportError:
            raise ImportError(
                "lxml package not found, please install it with " "`pip install lxml`"
            )
        super().__init__(web_path)
        self.filter_urls = filter_urls
        self.parsing_function = parsing_function or _default_parsing_function
        self.meta_function = meta_function or _default_meta_function
        self.blocksize = blocksize
        self.blocknum = blocknum
        self.is_local = is_local
[docs]    def parse_sitemap(self, soup: Any) -> List[dict]:
        """Parse sitemap xml and load into a list of dicts."""
        els = []
        for url in soup.find_all("url"):
            loc = url.find("loc")
            if not loc:
                continue
            # Strip leading and trailing whitespace and newlines
            loc_text = loc.text.strip()
            if self.filter_urls and not any(
                re.match(r, loc_text) for r in self.filter_urls
            ):
                continue
            els.append(
                {
                    tag: prop.text
                    for tag in ["loc", "lastmod", "changefreq", "priority"]
                    if (prop := url.find(tag))
                }
            )


if (prop := url.find(tag))
                }
            )
        for sitemap in soup.find_all("sitemap"):
            loc = sitemap.find("loc")
            if not loc:
                continue
            soup_child = self.scrape_all([loc.text], "xml")[0]
            els.extend(self.parse_sitemap(soup_child))
        return els
[docs]    def load(self) -> List[Document]:
        """Load sitemap."""
        if self.is_local:
            try:
                import bs4
            except ImportError:
                raise ImportError(
                    "beautifulsoup4 package not found, please install it"
                    " with `pip install beautifulsoup4`"
                )
            fp = open(self.web_path)
            soup = bs4.BeautifulSoup(fp, "xml")
        else:
            soup = self.scrape("xml")
        els = self.parse_sitemap(soup)
        if self.blocksize is not None:
            elblocks = list(_batch_block(els, self.blocksize))
            blockcount = len(elblocks)
            if blockcount - 1 < self.blocknum:
                raise ValueError(
                    "Selected sitemap does not contain enough blocks for given blocknum"
                )
            else:
                els = elblocks[self.blocknum]
        results = self.scrape_all([el["loc"].strip() for el in els if "loc" in el])
        return [
            Document(
                page_content=self.parsing_function(results[i]),
                metadata=self.meta_function(els[i], results[i]),
            )
            for i in range(len(results))
        ]


Source code for langchain.document_loaders.hugging_face_dataset
"""Loads HuggingFace datasets."""
from typing import Iterator, List, Mapping, Optional, Sequence, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class HuggingFaceDatasetLoader(BaseLoader):
    """Load Documents from the Hugging Face Hub."""
    def __init__(
        self,
        path: str,
        page_content_column: str = "text",
        name: Optional[str] = None,
        data_dir: Optional[str] = None,
        data_files: Optional[
            Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
        ] = None,
        cache_dir: Optional[str] = None,
        keep_in_memory: Optional[bool] = None,
        save_infos: bool = False,
        use_auth_token: Optional[Union[bool, str]] = None,
        num_proc: Optional[int] = None,
    ):
        """Initialize the HuggingFaceDatasetLoader.
        Args:
            path: Path or name of the dataset.
            page_content_column: Page content column name. Default is "text".
            name: Name of the dataset configuration.
            data_dir: Data directory of the dataset configuration.
            data_files: Path(s) to source data file(s).
            cache_dir: Directory to read/write data.
            keep_in_memory: Whether to copy the dataset in-memory.
            save_infos: Save the dataset information (checksums/size/splits/...).
              Default is False.
            use_auth_token: Bearer token for remote files on the Dataset Hub.
            num_proc: Number of processes.
        """
        self.path = path
        self.page_content_column = page_content_column


"""
        self.path = path
        self.page_content_column = page_content_column
        self.name = name
        self.data_dir = data_dir
        self.data_files = data_files
        self.cache_dir = cache_dir
        self.keep_in_memory = keep_in_memory
        self.save_infos = save_infos
        self.use_auth_token = use_auth_token
        self.num_proc = num_proc
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Load documents lazily."""
        try:
            from datasets import load_dataset
        except ImportError:
            raise ImportError(
                "Could not import datasets python package. "
                "Please install it with `pip install datasets`."
            )
        dataset = load_dataset(
            path=self.path,
            name=self.name,
            data_dir=self.data_dir,
            data_files=self.data_files,
            cache_dir=self.cache_dir,
            keep_in_memory=self.keep_in_memory,
            save_infos=self.save_infos,
            use_auth_token=self.use_auth_token,
            num_proc=self.num_proc,
        )
        yield from (
            Document(
                page_content=row.pop(self.page_content_column),
                metadata=row,
            )
            for key in dataset.keys()
            for row in dataset[key]
        )
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        return list(self.lazy_load())


Source code for langchain.document_loaders.gitbook
"""Loader that loads GitBook."""
from typing import Any, List, Optional
from urllib.parse import urljoin, urlparse
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class GitbookLoader(WebBaseLoader):
    """Load GitBook data.
    1. load from either a single page, or
    2. load all (relative) paths in the navbar.
    """
    def __init__(
        self,
        web_page: str,
        load_all_paths: bool = False,
        base_url: Optional[str] = None,
        content_selector: str = "main",
    ):
        """Initialize with web page and whether to load all paths.
        Args:
            web_page: The web page to load or the starting point from where
                relative paths are discovered.
            load_all_paths: If set to True, all relative paths in the navbar
                are loaded instead of only `web_page`.
            base_url: If `load_all_paths` is True, the relative paths are
                appended to this base url. Defaults to `web_page`.
            content_selector: The CSS selector for the content to load.
                Defaults to "main".
        """
        self.base_url = base_url or web_page
        if self.base_url.endswith("/"):
            self.base_url = self.base_url[:-1]
        if load_all_paths:
            # set web_path to the sitemap if we want to crawl all paths
            web_paths = f"{self.base_url}/sitemap.xml"
        else:
            web_paths = web_page
        super().__init__(web_paths)
        self.load_all_paths = load_all_paths
        self.content_selector = content_selector


self.load_all_paths = load_all_paths
        self.content_selector = content_selector
[docs]    def load(self) -> List[Document]:
        """Fetch text from one single GitBook page."""
        if self.load_all_paths:
            soup_info = self.scrape()
            relative_paths = self._get_paths(soup_info)
            urls = [urljoin(self.base_url, path) for path in relative_paths]
            soup_infos = self.scrape_all(urls)
            _documents = [
                self._get_document(soup_info, url)
                for soup_info, url in zip(soup_infos, urls)
            ]
        else:
            soup_info = self.scrape()
            _documents = [self._get_document(soup_info, self.web_path)]
        documents = [d for d in _documents if d]
        return documents
    def _get_document(
        self, soup: Any, custom_url: Optional[str] = None
    ) -> Optional[Document]:
        """Fetch content from page and return Document."""
        page_content_raw = soup.find(self.content_selector)
        if not page_content_raw:
            return None
        content = page_content_raw.get_text(separator="\n").strip()
        title_if_exists = page_content_raw.find("h1")
        title = title_if_exists.text if title_if_exists else ""
        metadata = {"source": custom_url or self.web_path, "title": title}
        return Document(page_content=content, metadata=metadata)
    def _get_paths(self, soup: Any) -> List[str]:
        """Fetch all relative paths in the navbar."""
        return [urlparse(loc.text).path for loc in soup.find_all("loc")]


Source code for langchain.document_loaders.tencent_cos_file
"""Loading logic for loading documents from Tencent Cloud COS file."""
import os
import tempfile
from typing import Any, Iterator, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class TencentCOSFileLoader(BaseLoader):
    """Loading logic for loading documents from Tencent Cloud COS."""
    def __init__(self, conf: Any, bucket: str, key: str):
        """Initialize with COS config, bucket and key name.
        :param conf(CosConfig): COS config.
        :param bucket(str): COS bucket.
        :param key(str): COS file key.
        """
        self.conf = conf
        self.bucket = bucket
        self.key = key
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load documents."""
        try:
            from qcloud_cos import CosS3Client
        except ImportError:
            raise ValueError(
                "Could not import cos-python-sdk-v5 python package. "
                "Please install it with `pip install cos-python-sdk-v5`."
            )
        # Initialise a client
        client = CosS3Client(self.conf)
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.bucket}/{self.key}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            # Download the file to a destination
            client.download_file(
                Bucket=self.bucket, Key=self.key, DestFilePath=file_path
            )
            loader = UnstructuredFileLoader(file_path)


)
            loader = UnstructuredFileLoader(file_path)
            # UnstructuredFileLoader not implement lazy_load yet
            return iter(loader.load())


Source code for langchain.document_loaders.image_captions
"""Loads image captions.
By default, the loader utilizes the pre-trained BLIP image captioning model.
https://huggingface.co/Salesforce/blip-image-captioning-base
"""
from typing import Any, List, Tuple, Union
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ImageCaptionLoader(BaseLoader):
    """Loads the captions of an image"""
    def __init__(
        self,
        path_images: Union[str, List[str]],
        blip_processor: str = "Salesforce/blip-image-captioning-base",
        blip_model: str = "Salesforce/blip-image-captioning-base",
    ):
        """
        Initialize with a list of image paths
        Args:
            path_images: A list of image paths.
            blip_processor: The name of the pre-trained BLIP processor.
            blip_model: The name of the pre-trained BLIP model.
        """
        if isinstance(path_images, str):
            self.image_paths = [path_images]
        else:
            self.image_paths = path_images
        self.blip_processor = blip_processor
        self.blip_model = blip_model
[docs]    def load(self) -> List[Document]:
        """
        Load from a list of image files
        """
        try:
            from transformers import BlipForConditionalGeneration, BlipProcessor
        except ImportError:
            raise ImportError(
                "`transformers` package not found, please install with "
                "`pip install transformers`."
            )
        processor = BlipProcessor.from_pretrained(self.blip_processor)
        model = BlipForConditionalGeneration.from_pretrained(self.blip_model)
        results = []


results = []
        for path_image in self.image_paths:
            caption, metadata = self._get_captions_and_metadata(
                model=model, processor=processor, path_image=path_image
            )
            doc = Document(page_content=caption, metadata=metadata)
            results.append(doc)
        return results
    def _get_captions_and_metadata(
        self, model: Any, processor: Any, path_image: str
    ) -> Tuple[str, dict]:
        """
        Helper function for getting the captions and metadata of an image
        """
        try:
            from PIL import Image
        except ImportError:
            raise ImportError(
                "`PIL` package not found, please install with `pip install pillow`"
            )
        try:
            if path_image.startswith("http://") or path_image.startswith("https://"):
                image = Image.open(requests.get(path_image, stream=True).raw).convert(
                    "RGB"
                )
            else:
                image = Image.open(path_image).convert("RGB")
        except Exception:
            raise ValueError(f"Could not get image data for {path_image}")
        inputs = processor(image, "an image of", return_tensors="pt")
        output = model.generate(**inputs)
        caption: str = processor.decode(output[0])
        metadata: dict = {"image_path": path_image}
        return caption, metadata


Source code for langchain.document_loaders.psychic
"""Loader that loads documents from Psychic.dev."""
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class PsychicLoader(BaseLoader):
    """Loader that loads documents from Psychic.dev."""
    def __init__(
        self, api_key: str, account_id: str, connector_id: Optional[str] = None
    ):
        """Initialize with API key, connector id, and account id."""
        try:
            from psychicapi import ConnectorId, Psychic  # noqa: F401
        except ImportError:
            raise ImportError(
                "`psychicapi` package not found, please run `pip install psychicapi`"
            )
        self.psychic = Psychic(secret_key=api_key)
        self.connector_id = ConnectorId(connector_id)
        self.account_id = account_id
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        psychic_docs = self.psychic.get_documents(
            connector_id=self.connector_id, account_id=self.account_id
        )
        return [
            Document(
                page_content=doc["content"],
                metadata={"title": doc["title"], "source": doc["uri"]},
            )
            for doc in psychic_docs.documents
        ]


Source code for langchain.document_loaders.reddit
"""Reddit document loader."""
from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import praw
def _dependable_praw_import() -> praw:
    try:
        import praw
    except ImportError:
        raise ValueError(
            "praw package not found, please install it with `pip install praw`"
        )
    return praw
[docs]class RedditPostsLoader(BaseLoader):
    """Reddit posts loader.
    Read posts on a subreddit.
    First you need to go to
    https://www.reddit.com/prefs/apps/
    and create your application
    """
    def __init__(
        self,
        client_id: str,
        client_secret: str,
        user_agent: str,
        search_queries: Sequence[str],
        mode: str,
        categories: Sequence[str] = ["new"],
        number_posts: Optional[int] = 10,
    ):
        self.client_id = client_id
        self.client_secret = client_secret
        self.user_agent = user_agent
        self.search_queries = search_queries
        self.mode = mode
        self.categories = categories
        self.number_posts = number_posts
[docs]    def load(self) -> List[Document]:
        """Load reddits."""
        praw = _dependable_praw_import()
        reddit = praw.Reddit(
            client_id=self.client_id,
            client_secret=self.client_secret,
            user_agent=self.user_agent,
        )
        results: List[Document] = []
        if self.mode == "subreddit":
            for search_query in self.search_queries:


if self.mode == "subreddit":
            for search_query in self.search_queries:
                for category in self.categories:
                    docs = self._subreddit_posts_loader(
                        search_query=search_query, category=category, reddit=reddit
                    )
                    results.extend(docs)
        elif self.mode == "username":
            for search_query in self.search_queries:
                for category in self.categories:
                    docs = self._user_posts_loader(
                        search_query=search_query, category=category, reddit=reddit
                    )
                    results.extend(docs)
        else:
            raise ValueError(
                "mode not correct, please enter 'username' or 'subreddit' as mode"
            )
        return results
    def _subreddit_posts_loader(
        self, search_query: str, category: str, reddit: praw.reddit.Reddit
    ) -> Iterable[Document]:
        subreddit = reddit.subreddit(search_query)
        method = getattr(subreddit, category)
        cat_posts = method(limit=self.number_posts)
        """Format reddit posts into a string."""
        for post in cat_posts:
            metadata = {
                "post_subreddit": post.subreddit_name_prefixed,
                "post_category": category,
                "post_title": post.title,
                "post_score": post.score,
                "post_id": post.id,
                "post_url": post.url,
                "post_author": post.author,
            }
            yield Document(
                page_content=post.selftext,
                metadata=metadata,
            )
    def _user_posts_loader(
        self, search_query: str, category: str, reddit: praw.reddit.Reddit
    ) -> Iterable[Document]:
        user = reddit.redditor(search_query)
        method = getattr(user.submissions, category)


method = getattr(user.submissions, category)
        cat_posts = method(limit=self.number_posts)
        """Format reddit posts into a string."""
        for post in cat_posts:
            metadata = {
                "post_subreddit": post.subreddit_name_prefixed,
                "post_category": category,
                "post_title": post.title,
                "post_score": post.score,
                "post_id": post.id,
                "post_url": post.url,
                "post_author": post.author,
            }
            yield Document(
                page_content=post.selftext,
                metadata=metadata,
            )


Source code for langchain.document_loaders.airbyte_json
"""Loader that loads local airbyte json files."""
import json
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import stringify_dict
[docs]class AirbyteJSONLoader(BaseLoader):
    """Loader that loads local airbyte json files."""
    def __init__(self, file_path: str):
        """Initialize with a file path. This should start with '/tmp/airbyte_local/'."""
        self.file_path = file_path
        """Path to the directory containing the json files."""
[docs]    def load(self) -> List[Document]:
        text = ""
        for line in open(self.file_path, "r"):
            data = json.loads(line)["_airbyte_data"]
            text += stringify_dict(data)
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.evernote
"""Load documents from Evernote.
https://gist.github.com/foxmask/7b29c43a161e001ff04afdb2f181e31c
"""
import hashlib
import logging
from base64 import b64decode
from time import strptime
from typing import Any, Dict, Iterator, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class EverNoteLoader(BaseLoader):
    """EverNote Loader.
    Loads an EverNote notebook export file e.g. my_notebook.enex into Documents.
    Instructions on producing this file can be found at
    https://help.evernote.com/hc/en-us/articles/209005557-Export-notes-and-notebooks-as-ENEX-or-HTML
    Currently only the plain text in the note is extracted and stored as the contents
    of the Document, any non content metadata (e.g. 'author', 'created', 'updated' etc.
    but not 'content-raw' or 'resource') tags on the note will be extracted and stored
    as metadata on the Document.
    Args:
        file_path (str): The path to the notebook export with a .enex extension
        load_single_document (bool): Whether or not to concatenate the content of all
            notes into a single long Document.
        If this is set to True (default) then the only metadata on the document will be
            the 'source' which contains the file name of the export.
    """  # noqa: E501
    def __init__(self, file_path: str, load_single_document: bool = True):
        """Initialize with file path."""
        self.file_path = file_path
        self.load_single_document = load_single_document


self.file_path = file_path
        self.load_single_document = load_single_document
[docs]    def load(self) -> List[Document]:
        """Load documents from EverNote export file."""
        documents = [
            Document(
                page_content=note["content"],
                metadata={
                    **{
                        key: value
                        for key, value in note.items()
                        if key not in ["content", "content-raw", "resource"]
                    },
                    **{"source": self.file_path},
                },
            )
            for note in self._parse_note_xml(self.file_path)
            if note.get("content") is not None
        ]
        if not self.load_single_document:
            return documents
        return [
            Document(
                page_content="".join([document.page_content for document in documents]),
                metadata={"source": self.file_path},
            )
        ]
    @staticmethod
    def _parse_content(content: str) -> str:
        try:
            import html2text
            return html2text.html2text(content).strip()
        except ImportError as e:
            logging.error(
                "Could not import `html2text`. Although it is not a required package "
                "to use Langchain, using the EverNote loader requires `html2text`. "
                "Please install `html2text` via `pip install html2text` and try again."
            )
            raise e
    @staticmethod
    def _parse_resource(resource: list) -> dict:
        rsc_dict: Dict[str, Any] = {}
        for elem in resource:
            if elem.tag == "data":
                # Sometimes elem.text is None
                rsc_dict[elem.tag] = b64decode(elem.text) if elem.text else b""


rsc_dict["hash"] = hashlib.md5(rsc_dict[elem.tag]).hexdigest()
            else:
                rsc_dict[elem.tag] = elem.text
        return rsc_dict
    @staticmethod
    def _parse_note(note: List, prefix: Optional[str] = None) -> dict:
        note_dict: Dict[str, Any] = {}
        resources = []
        def add_prefix(element_tag: str) -> str:
            if prefix is None:
                return element_tag
            return f"{prefix}.{element_tag}"
        for elem in note:
            if elem.tag == "content":
                note_dict[elem.tag] = EverNoteLoader._parse_content(elem.text)
                # A copy of original content
                note_dict["content-raw"] = elem.text
            elif elem.tag == "resource":
                resources.append(EverNoteLoader._parse_resource(elem))
            elif elem.tag == "created" or elem.tag == "updated":
                note_dict[elem.tag] = strptime(elem.text, "%Y%m%dT%H%M%SZ")
            elif elem.tag == "note-attributes":
                additional_attributes = EverNoteLoader._parse_note(
                    elem, elem.tag
                )  # Recursively enter the note-attributes tag
                note_dict.update(additional_attributes)
            else:
                note_dict[elem.tag] = elem.text
        if len(resources) > 0:
            note_dict["resource"] = resources
        return {add_prefix(key): value for key, value in note_dict.items()}
    @staticmethod
    def _parse_note_xml(xml_file: str) -> Iterator[Dict[str, Any]]:
        """Parse Evernote xml."""
        # Without huge_tree set to True, parser may complain about huge text node


# Without huge_tree set to True, parser may complain about huge text node
        # Try to recover, because there may be " ", which will cause
        # "XMLSyntaxError: Entity 'nbsp' not defined"
        try:
            from lxml import etree
        except ImportError as e:
            logging.error(
                "Could not import `lxml`. Although it is not a required package to use "
                "Langchain, using the EverNote loader requires `lxml`. Please install "
                "`lxml` via `pip install lxml` and try again."
            )
            raise e
        context = etree.iterparse(
            xml_file, encoding="utf-8", strip_cdata=False, huge_tree=True, recover=True
        )
        for action, elem in context:
            if elem.tag == "note":
                yield EverNoteLoader._parse_note(elem)


Source code for langchain.document_loaders.toml
import json
from pathlib import Path
from typing import Iterator, List, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class TomlLoader(BaseLoader):
    """
    A TOML document loader that inherits from the BaseLoader class.
    This class can be initialized with either a single source file or a source
    directory containing TOML files.
    """
    def __init__(self, source: Union[str, Path]):
        """Initialize the TomlLoader with a source file or directory."""
        self.source = Path(source)
[docs]    def load(self) -> List[Document]:
        """Load and return all documents."""
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazily load the TOML documents from the source file or directory."""
        import tomli
        if self.source.is_file() and self.source.suffix == ".toml":
            files = [self.source]
        elif self.source.is_dir():
            files = list(self.source.glob("**/*.toml"))
        else:
            raise ValueError("Invalid source path or file type")
        for file_path in files:
            with file_path.open("r", encoding="utf-8") as file:
                content = file.read()
                try:
                    data = tomli.loads(content)
                    doc = Document(
                        page_content=json.dumps(data),
                        metadata={"source": str(file_path)},
                    )
                    yield doc
                except tomli.TOMLDecodeError as e:
                    print(f"Error parsing TOML file {file_path}: {e}")


Source code for langchain.document_loaders.fauna
from typing import Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class FaunaLoader(BaseLoader):
    """FaunaDB Loader.
    Attributes:
        query (str): The FQL query string to execute.
        page_content_field (str): The field that contains the content of each page.
        secret (str): The secret key for authenticating to FaunaDB.
        metadata_fields (Optional[Sequence[str]]):
            Optional list of field names to include in metadata.
    """
    def __init__(
        self,
        query: str,
        page_content_field: str,
        secret: str,
        metadata_fields: Optional[Sequence[str]] = None,
    ):
        self.query = query
        self.page_content_field = page_content_field
        self.secret = secret
        self.metadata_fields = metadata_fields
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        try:
            from fauna import Page, fql
            from fauna.client import Client
            from fauna.encoding import QuerySuccess
        except ImportError:
            raise ImportError(
                "Could not import fauna python package. "
                "Please install it with `pip install fauna`."
            )
        # Create Fauna Client
        client = Client(secret=self.secret)
        # Run FQL Query
        response: QuerySuccess = client.query(fql(self.query))
        page: Page = response.data
        for result in page:
            if result is not None:
                document_dict = dict(result.items())
                page_content = ""


document_dict = dict(result.items())
                page_content = ""
                for key, value in document_dict.items():
                    if key == self.page_content_field:
                        page_content = value
                document: Document = Document(
                    page_content=page_content,
                    metadata={"id": result.id, "ts": result.ts},
                )
                yield document
        if page.after is not None:
            yield Document(
                page_content="Next Page Exists",
                metadata={"after": page.after},
            )


Source code for langchain.document_loaders.embaas
import base64
import warnings
from typing import Any, Dict, Iterator, List, Optional
import requests
from pydantic import BaseModel, root_validator, validator
from typing_extensions import NotRequired, TypedDict
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseBlobParser, BaseLoader
from langchain.document_loaders.blob_loaders import Blob
from langchain.text_splitter import TextSplitter
from langchain.utils import get_from_dict_or_env
EMBAAS_DOC_API_URL = "https://api.embaas.io/v1/document/extract-text/bytes/"
[docs]class EmbaasDocumentExtractionParameters(TypedDict):
    """Parameters for the embaas document extraction API."""
    mime_type: NotRequired[str]
    """The mime type of the document."""
    file_extension: NotRequired[str]
    """The file extension of the document."""
    file_name: NotRequired[str]
    """The file name of the document."""
    should_chunk: NotRequired[bool]
    """Whether to chunk the document into pages."""
    chunk_size: NotRequired[int]
    """The maximum size of the text chunks."""
    chunk_overlap: NotRequired[int]
    """The maximum overlap allowed between chunks."""
    chunk_splitter: NotRequired[str]
    """The text splitter class name for creating chunks."""
    separators: NotRequired[List[str]]
    """The separators for chunks."""
    should_embed: NotRequired[bool]
    """Whether to create embeddings for the document in the response."""
    model: NotRequired[str]
    """The model to pass to the Embaas document extraction API."""
    instruction: NotRequired[str]
    """The instruction to pass to the Embaas document extraction API."""


"""The instruction to pass to the Embaas document extraction API."""
[docs]class EmbaasDocumentExtractionPayload(EmbaasDocumentExtractionParameters):
    """Payload for the Embaas document extraction API."""
    bytes: str
    """The base64 encoded bytes of the document to extract text from."""
[docs]class BaseEmbaasLoader(BaseModel):
    """Base class for embedding a model into an Embaas document extraction API."""
    embaas_api_key: Optional[str] = None
    """The API key for the embaas document extraction API."""
    api_url: str = EMBAAS_DOC_API_URL
    """The URL of the embaas document extraction API."""
    params: EmbaasDocumentExtractionParameters = EmbaasDocumentExtractionParameters()
    """Additional parameters to pass to the embaas document extraction API."""
[docs]    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        embaas_api_key = get_from_dict_or_env(
            values, "embaas_api_key", "EMBAAS_API_KEY"
        )
        values["embaas_api_key"] = embaas_api_key
        return values
[docs]class EmbaasBlobLoader(BaseEmbaasLoader, BaseBlobParser):
    """Embaas's document byte loader.
    To use, you should have the
    environment variable ``EMBAAS_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            # Default parsing
            from langchain.document_loaders.embaas import EmbaasBlobLoader
            loader = EmbaasBlobLoader()


loader = EmbaasBlobLoader()
            blob = Blob.from_path(path="example.mp3")
            documents = loader.parse(blob=blob)
            # Custom api parameters (create embeddings automatically)
            from langchain.document_loaders.embaas import EmbaasBlobLoader
            loader = EmbaasBlobLoader(
                params={
                    "should_embed": True,
                    "model": "e5-large-v2",
                    "chunk_size": 256,
                    "chunk_splitter": "CharacterTextSplitter"
                }
            )
            blob = Blob.from_path(path="example.pdf")
            documents = loader.parse(blob=blob)
    """
[docs]    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        """Parses the blob lazily.
        Args:
            blob: The blob to parse.
        """
        yield from self._get_documents(blob=blob)
    @staticmethod
    def _api_response_to_documents(chunks: List[Dict[str, Any]]) -> List[Document]:
        """Convert the API response to a list of documents."""
        docs = []
        for chunk in chunks:
            metadata = chunk["metadata"]
            if chunk.get("embedding", None) is not None:
                metadata["embedding"] = chunk["embedding"]
            doc = Document(page_content=chunk["text"], metadata=metadata)
            docs.append(doc)
        return docs
    def _generate_payload(self, blob: Blob) -> EmbaasDocumentExtractionPayload:
        """Generates payload for the API request."""
        base64_byte_str = base64.b64encode(blob.as_bytes()).decode()
        payload: EmbaasDocumentExtractionPayload = EmbaasDocumentExtractionPayload(
            bytes=base64_byte_str,


bytes=base64_byte_str,
            # Workaround for mypy issue: https://github.com/python/mypy/issues/9408
            # type: ignore
            **self.params,
        )
        if blob.mimetype is not None and payload.get("mime_type", None) is None:
            payload["mime_type"] = blob.mimetype
        return payload
    def _handle_request(
        self, payload: EmbaasDocumentExtractionPayload
    ) -> List[Document]:
        """Sends a request to the embaas API and handles the response."""
        headers = {
            "Authorization": f"Bearer {self.embaas_api_key}",
            "Content-Type": "application/json",
        }
        response = requests.post(self.api_url, headers=headers, json=payload)
        response.raise_for_status()
        parsed_response = response.json()
        return EmbaasBlobLoader._api_response_to_documents(
            chunks=parsed_response["data"]["chunks"]
        )
    def _get_documents(self, blob: Blob) -> Iterator[Document]:
        """Get the documents from the blob."""
        payload = self._generate_payload(blob=blob)
        try:
            documents = self._handle_request(payload=payload)
        except requests.exceptions.RequestException as e:
            if e.response is None or not e.response.text:
                raise ValueError(
                    f"Error raised by embaas document text extraction API: {e}"
                )
            parsed_response = e.response.json()
            if "message" in parsed_response:
                raise ValueError(
                    f"Validation Error raised by embaas document text extraction API:"
                    f" {parsed_response['message']}"
                )
            raise
        yield from documents


)
            raise
        yield from documents
[docs]class EmbaasLoader(BaseEmbaasLoader, BaseLoader):
    """Embaas's document loader.
    To use, you should have the
    environment variable ``EMBAAS_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            # Default parsing
            from langchain.document_loaders.embaas import EmbaasLoader
            loader = EmbaasLoader(file_path="example.mp3")
            documents = loader.load()
            # Custom api parameters (create embeddings automatically)
            from langchain.document_loaders.embaas import EmbaasBlobLoader
            loader = EmbaasBlobLoader(
                file_path="example.pdf",
                params={
                    "should_embed": True,
                    "model": "e5-large-v2",
                    "chunk_size": 256,
                    "chunk_splitter": "CharacterTextSplitter"
                }
            )
            documents = loader.load()
    """
    file_path: str
    """The path to the file to load."""
    blob_loader: Optional[EmbaasBlobLoader]
    """The blob loader to use. If not provided, a default one will be created."""
[docs]    @validator("blob_loader", always=True)
    def validate_blob_loader(
        cls, v: EmbaasBlobLoader, values: Dict
    ) -> EmbaasBlobLoader:
        return v or EmbaasBlobLoader(
            embaas_api_key=values["embaas_api_key"],
            api_url=values["api_url"],
            params=values["params"],
        )
[docs]    def lazy_load(self) -> Iterator[Document]:


)
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load the documents from the file path lazily."""
        blob = Blob.from_path(path=self.file_path)
        assert self.blob_loader is not None
        # Should never be None, but mypy doesn't know that.
        yield from self.blob_loader.lazy_parse(blob=blob)
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def load_and_split(
        self, text_splitter: Optional[TextSplitter] = None
    ) -> List[Document]:
        if self.params.get("should_embed", False):
            warnings.warn(
                "Embeddings are not supported with load_and_split."
                " Use the API splitter to properly generate embeddings."
                " For more information see embaas.io docs."
            )
        return super().load_and_split(text_splitter=text_splitter)


Source code for langchain.document_loaders.docugami
"""Loads processed documents from Docugami."""
import io
import logging
import os
import re
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Sequence, Union
import requests
from pydantic import BaseModel, root_validator
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
TD_NAME = "{http://www.w3.org/1999/xhtml}td"
TABLE_NAME = "{http://www.w3.org/1999/xhtml}table"
XPATH_KEY = "xpath"
DOCUMENT_ID_KEY = "id"
DOCUMENT_NAME_KEY = "name"
STRUCTURE_KEY = "structure"
TAG_KEY = "tag"
PROJECTS_KEY = "projects"
DEFAULT_API_ENDPOINT = "https://api.docugami.com/v1preview1"
logger = logging.getLogger(__name__)
[docs]class DocugamiLoader(BaseLoader, BaseModel):
    """Loads processed docs from Docugami.
    To use, you should have the ``lxml`` python package installed.
    """
    api: str = DEFAULT_API_ENDPOINT
    """The Docugami API endpoint to use."""
    access_token: Optional[str] = os.environ.get("DOCUGAMI_API_KEY")
    """The Docugami API access token to use."""
    docset_id: Optional[str]
    """The Docugami API docset ID to use."""
    document_ids: Optional[Sequence[str]]
    """The Docugami API document IDs to use."""
    file_paths: Optional[Sequence[Union[Path, str]]]
    """The local file paths to use."""
    min_chunk_size: int = 32  # appended to the next chunk to avoid over-chunking


"""The minimum chunk size to use when parsing DGML. Defaults to 32."""
[docs]    @root_validator
    def validate_local_or_remote(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that either local file paths are given, or remote API docset ID.
        Args:
            values: The values to validate.
        Returns:
            The validated values.
        """
        if values.get("file_paths") and values.get("docset_id"):
            raise ValueError("Cannot specify both file_paths and remote API docset_id")
        if not values.get("file_paths") and not values.get("docset_id"):
            raise ValueError("Must specify either file_paths or remote API docset_id")
        if values.get("docset_id") and not values.get("access_token"):
            raise ValueError("Must specify access token if using remote API docset_id")
        return values
    def _parse_dgml(
        self, document: Mapping, content: bytes, doc_metadata: Optional[Mapping] = None
    ) -> List[Document]:
        """Parse a single DGML document into a list of Documents."""
        try:
            from lxml import etree
        except ImportError:
            raise ImportError(
                "Could not import lxml python package. "
                "Please install it with `pip install lxml`."
            )
        # helpers
        def _xpath_qname_for_chunk(chunk: Any) -> str:
            """Get the xpath qname for a chunk."""
            qname = f"{chunk.prefix}:{chunk.tag.split('}')[-1]}"
            parent = chunk.getparent()
            if parent is not None:
                doppelgangers = [x for x in parent if x.tag == chunk.tag]


doppelgangers = [x for x in parent if x.tag == chunk.tag]
                if len(doppelgangers) > 1:
                    idx_of_self = doppelgangers.index(chunk)
                    qname = f"{qname}[{idx_of_self + 1}]"
            return qname
        def _xpath_for_chunk(chunk: Any) -> str:
            """Get the xpath for a chunk."""
            ancestor_chain = chunk.xpath("ancestor-or-self::*")
            return "/" + "/".join(_xpath_qname_for_chunk(x) for x in ancestor_chain)
        def _structure_value(node: Any) -> str:
            """Get the structure value for a node."""
            structure = (
                "table"
                if node.tag == TABLE_NAME
                else node.attrib["structure"]
                if "structure" in node.attrib
                else None
            )
            return structure
        def _is_structural(node: Any) -> bool:
            """Check if a node is structural."""
            return _structure_value(node) is not None
        def _is_heading(node: Any) -> bool:
            """Check if a node is a heading."""
            structure = _structure_value(node)
            return structure is not None and structure.lower().startswith("h")
        def _get_text(node: Any) -> str:
            """Get the text of a node."""
            return " ".join(node.itertext()).strip()
        def _has_structural_descendant(node: Any) -> bool:
            """Check if a node has a structural descendant."""
            for child in node:
                if _is_structural(child) or _has_structural_descendant(child):
                    return True
            return False
        def _leaf_structural_nodes(node: Any) -> List:


return False
        def _leaf_structural_nodes(node: Any) -> List:
            """Get the leaf structural nodes of a node."""
            if _is_structural(node) and not _has_structural_descendant(node):
                return [node]
            else:
                leaf_nodes = []
                for child in node:
                    leaf_nodes.extend(_leaf_structural_nodes(child))
                return leaf_nodes
        def _create_doc(node: Any, text: str) -> Document:
            """Create a Document from a node and text."""
            metadata = {
                XPATH_KEY: _xpath_for_chunk(node),
                DOCUMENT_ID_KEY: document["id"],
                DOCUMENT_NAME_KEY: document["name"],
                STRUCTURE_KEY: node.attrib.get("structure", ""),
                TAG_KEY: re.sub(r"\{.*\}", "", node.tag),
            }
            if doc_metadata:
                metadata.update(doc_metadata)
            return Document(
                page_content=text,
                metadata=metadata,
            )
        # parse the tree and return chunks
        tree = etree.parse(io.BytesIO(content))
        root = tree.getroot()
        chunks: List[Document] = []
        prev_small_chunk_text = None
        for node in _leaf_structural_nodes(root):
            text = _get_text(node)
            if prev_small_chunk_text:
                text = prev_small_chunk_text + " " + text
                prev_small_chunk_text = None
            if _is_heading(node) or len(text) < self.min_chunk_size:
                # Save headings or other small chunks to be appended to the next chunk
                prev_small_chunk_text = text
            else:
                chunks.append(_create_doc(node, text))
        if prev_small_chunk_text and len(chunks) > 0:


if prev_small_chunk_text and len(chunks) > 0:
            # small chunk at the end left over, just append to last chunk
            chunks[-1].page_content += " " + prev_small_chunk_text
        return chunks
    def _document_details_for_docset_id(self, docset_id: str) -> List[Dict]:
        """Gets all document details for the given docset ID"""
        url = f"{self.api}/docsets/{docset_id}/documents"
        all_documents = []
        while url:
            response = requests.get(
                url,
                headers={"Authorization": f"Bearer {self.access_token}"},
            )
            if response.ok:
                data = response.json()
                all_documents.extend(data["documents"])
                url = data.get("next", None)
            else:
                raise Exception(
                    f"Failed to download {url} (status: {response.status_code})"
                )
        return all_documents
    def _project_details_for_docset_id(self, docset_id: str) -> List[Dict]:
        """Gets all project details for the given docset ID"""
        url = f"{self.api}/projects?docset.id={docset_id}"
        all_projects = []
        while url:
            response = requests.request(
                "GET",
                url,
                headers={"Authorization": f"Bearer {self.access_token}"},
                data={},
            )
            if response.ok:
                data = response.json()
                all_projects.extend(data["projects"])
                url = data.get("next", None)
            else:
                raise Exception(
                    f"Failed to download {url} (status: {response.status_code})"
                )
        return all_projects


)
        return all_projects
    def _metadata_for_project(self, project: Dict) -> Dict:
        """Gets project metadata for all files"""
        project_id = project.get("id")
        url = f"{self.api}/projects/{project_id}/artifacts/latest"
        all_artifacts = []
        while url:
            response = requests.request(
                "GET",
                url,
                headers={"Authorization": f"Bearer {self.access_token}"},
                data={},
            )
            if response.ok:
                data = response.json()
                all_artifacts.extend(data["artifacts"])
                url = data.get("next", None)
            else:
                raise Exception(
                    f"Failed to download {url} (status: {response.status_code})"
                )
        per_file_metadata = {}
        for artifact in all_artifacts:
            artifact_name = artifact.get("name")
            artifact_url = artifact.get("url")
            artifact_doc = artifact.get("document")
            if artifact_name == "report-values.xml" and artifact_url and artifact_doc:
                doc_id = artifact_doc["id"]
                metadata: Dict = {}
                # the evaluated XML for each document is named after the project
                response = requests.request(
                    "GET",
                    f"{artifact_url}/content",
                    headers={"Authorization": f"Bearer {self.access_token}"},
                    data={},
                )
                if response.ok:
                    try:
                        from lxml import etree
                    except ImportError:
                        raise ImportError(
                            "Could not import lxml python package. "
                            "Please install it with `pip install lxml`."
                        )
                    artifact_tree = etree.parse(io.BytesIO(response.content))
                    artifact_root = artifact_tree.getroot()
                    ns = artifact_root.nsmap


artifact_root = artifact_tree.getroot()
                    ns = artifact_root.nsmap
                    entries = artifact_root.xpath("//pr:Entry", namespaces=ns)
                    for entry in entries:
                        heading = entry.xpath("./pr:Heading", namespaces=ns)[0].text
                        value = " ".join(
                            entry.xpath("./pr:Value", namespaces=ns)[0].itertext()
                        ).strip()
                        metadata[heading] = value
                    per_file_metadata[doc_id] = metadata
                else:
                    raise Exception(
                        f"Failed to download {artifact_url}/content "
                        + "(status: {response.status_code})"
                    )
        return per_file_metadata
    def _load_chunks_for_document(
        self, docset_id: str, document: Dict, doc_metadata: Optional[Dict] = None
    ) -> List[Document]:
        """Load chunks for a document."""
        document_id = document["id"]
        url = f"{self.api}/docsets/{docset_id}/documents/{document_id}/dgml"
        response = requests.request(
            "GET",
            url,
            headers={"Authorization": f"Bearer {self.access_token}"},
            data={},
        )
        if response.ok:
            return self._parse_dgml(document, response.content, doc_metadata)
        else:
            raise Exception(
                f"Failed to download {url} (status: {response.status_code})"
            )
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        chunks: List[Document] = []
        if self.access_token and self.docset_id:
            # remote mode
            _document_details = self._document_details_for_docset_id(self.docset_id)
            if self.document_ids:


if self.document_ids:
                _document_details = [
                    d for d in _document_details if d["id"] in self.document_ids
                ]
            _project_details = self._project_details_for_docset_id(self.docset_id)
            combined_project_metadata = {}
            if _project_details:
                # if there are any projects for this docset, load project metadata
                for project in _project_details:
                    metadata = self._metadata_for_project(project)
                    combined_project_metadata.update(metadata)
            for doc in _document_details:
                doc_metadata = combined_project_metadata.get(doc["id"])
                chunks += self._load_chunks_for_document(
                    self.docset_id, doc, doc_metadata
                )
        elif self.file_paths:
            # local mode (for integration testing, or pre-downloaded XML)
            for path in self.file_paths:
                path = Path(path)
                with open(path, "rb") as file:
                    chunks += self._parse_dgml(
                        {
                            DOCUMENT_ID_KEY: path.name,
                            DOCUMENT_NAME_KEY: path.name,
                        },
                        file.read(),
                    )
        return chunks


Source code for langchain.document_loaders.text
import logging
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.helpers import detect_file_encodings
logger = logging.getLogger(__name__)
[docs]class TextLoader(BaseLoader):
    """Load text files.
    Args:
        file_path: Path to the file to load.
        encoding: File encoding to use. If `None`, the file will be loaded
        with the default system encoding.
        autodetect_encoding: Whether to try to autodetect the file encoding
            if the specified encoding fails.
    """
    def __init__(
        self,
        file_path: str,
        encoding: Optional[str] = None,
        autodetect_encoding: bool = False,
    ):
        """Initialize with file path."""
        self.file_path = file_path
        self.encoding = encoding
        self.autodetect_encoding = autodetect_encoding
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        text = ""
        try:
            with open(self.file_path, encoding=self.encoding) as f:
                text = f.read()
        except UnicodeDecodeError as e:
            if self.autodetect_encoding:
                detected_encodings = detect_file_encodings(self.file_path)
                for encoding in detected_encodings:
                    logger.debug("Trying encoding: ", encoding.encoding)
                    try:
                        with open(self.file_path, encoding=encoding.encoding) as f:
                            text = f.read()
                        break
                    except UnicodeDecodeError:
                        continue
            else:
                raise RuntimeError(f"Error loading {self.file_path}") from e
        except Exception as e:


except Exception as e:
            raise RuntimeError(f"Error loading {self.file_path}") from e
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.blockchain
import os
import re
import time
from enum import Enum
from typing import List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class BlockchainType(Enum):
    """Enumerator of the supported blockchains."""
    ETH_MAINNET = "eth-mainnet"
    ETH_GOERLI = "eth-goerli"
    POLYGON_MAINNET = "polygon-mainnet"
    POLYGON_MUMBAI = "polygon-mumbai"
[docs]class BlockchainDocumentLoader(BaseLoader):
    """Loads elements from a blockchain smart contract into Langchain documents.
    The supported blockchains are: Ethereum mainnet, Ethereum Goerli testnet,
    Polygon mainnet, and Polygon Mumbai testnet.
    If no BlockchainType is specified, the default is Ethereum mainnet.
    The Loader uses the Alchemy API to interact with the blockchain.
    ALCHEMY_API_KEY environment variable must be set to use this loader.
    The API returns 100 NFTs per request and can be paginated using the
    startToken parameter.
    If get_all_tokens is set to True, the loader will get all tokens
    on the contract.  Note that for contracts with a large number of tokens,
    this may take a long time (e.g. 10k tokens is 100 requests).
    Default value is false for this reason.
    The max_execution_time (sec) can be set to limit the execution time
    of the loader.
    Future versions of this loader can:
        - Support additional Alchemy APIs (e.g. getTransactions, etc.)
        - Support additional blockain APIs (e.g. Infura, Opensea, etc.)
    """
    def __init__(


"""
    def __init__(
        self,
        contract_address: str,
        blockchainType: BlockchainType = BlockchainType.ETH_MAINNET,
        api_key: str = "docs-demo",
        startToken: str = "",
        get_all_tokens: bool = False,
        max_execution_time: Optional[int] = None,
    ):
        """
        Args:
            contract_address: The address of the smart contract.
            blockchainType: The blockchain type.
            api_key: The Alchemy API key.
            startToken: The start token for pagination.
            get_all_tokens: Whether to get all tokens on the contract.
            max_execution_time: The maximum execution time (sec).
        """
        self.contract_address = contract_address
        self.blockchainType = blockchainType.value
        self.api_key = os.environ.get("ALCHEMY_API_KEY") or api_key
        self.startToken = startToken
        self.get_all_tokens = get_all_tokens
        self.max_execution_time = max_execution_time
        if not self.api_key:
            raise ValueError("Alchemy API key not provided.")
        if not re.match(r"^0x[a-fA-F0-9]{40}$", self.contract_address):
            raise ValueError(f"Invalid contract address {self.contract_address}")
[docs]    def load(self) -> List[Document]:
        result = []
        current_start_token = self.startToken
        start_time = time.time()
        while True:
            url = (
                f"https://{self.blockchainType}.g.alchemy.com/nft/v2/"
                f"{self.api_key}/getNFTsForCollection?withMetadata="
                f"True&contractAddress={self.contract_address}"
                f"&startToken={current_start_token}"
            )
            response = requests.get(url)


)
            response = requests.get(url)
            if response.status_code != 200:
                raise ValueError(
                    f"Request failed with status code {response.status_code}"
                )
            items = response.json()["nfts"]
            if not items:
                break
            for item in items:
                content = str(item)
                tokenId = item["id"]["tokenId"]
                metadata = {
                    "source": self.contract_address,
                    "blockchain": self.blockchainType,
                    "tokenId": tokenId,
                }
                result.append(Document(page_content=content, metadata=metadata))
            # exit after the first API call if get_all_tokens is False
            if not self.get_all_tokens:
                break
            # get the start token for the next API call from the last item in array
            current_start_token = self._get_next_tokenId(result[-1].metadata["tokenId"])
            if (
                self.max_execution_time is not None
                and (time.time() - start_time) > self.max_execution_time
            ):
                raise RuntimeError("Execution time exceeded the allowed time limit.")
        if not result:
            raise ValueError(
                f"No NFTs found for contract address {self.contract_address}"
            )
        return result
    # add one to the tokenId, ensuring the correct tokenId format is used
    def _get_next_tokenId(self, tokenId: str) -> str:
        value_type = self._detect_value_type(tokenId)
        if value_type == "hex_0x":
            value_int = int(tokenId, 16)
        elif value_type == "hex_0xbf":
            value_int = int(tokenId[2:], 16)
        else:
            value_int = int(tokenId)
        result = value_int + 1


value_int = int(tokenId)
        result = value_int + 1
        if value_type == "hex_0x":
            return "0x" + format(result, "0" + str(len(tokenId) - 2) + "x")
        elif value_type == "hex_0xbf":
            return "0xbf" + format(result, "0" + str(len(tokenId) - 4) + "x")
        else:
            return str(result)
    # A smart contract can use different formats for the tokenId
    @staticmethod
    def _detect_value_type(tokenId: str) -> str:
        if isinstance(tokenId, int):
            return "int"
        elif tokenId.startswith("0x"):
            return "hex_0x"
        elif tokenId.startswith("0xbf"):
            return "hex_0xbf"
        else:
            return "hex_0xbf"


Source code for langchain.document_loaders.html_bs
"""Loader that uses bs4 to load HTML files, enriching metadata with page title."""
import logging
from typing import Dict, List, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class BSHTMLLoader(BaseLoader):
    """Loader that uses beautiful soup to parse HTML files."""
    def __init__(
        self,
        file_path: str,
        open_encoding: Union[str, None] = None,
        bs_kwargs: Union[dict, None] = None,
        get_text_separator: str = "",
    ) -> None:
        """Initialise with path, and optionally, file encoding to use, and any kwargs
        to pass to the BeautifulSoup object.
        Args:
            file_path: The path to the file to load.
            open_encoding: The encoding to use when opening the file.
            bs_kwargs: Any kwargs to pass to the BeautifulSoup object.
            get_text_separator: The separator to use when calling get_text on the soup.
        """
        try:
            import bs4  # noqa:F401
        except ImportError:
            raise ImportError(
                "beautifulsoup4 package not found, please install it with "
                "`pip install beautifulsoup4`"
            )
        self.file_path = file_path
        self.open_encoding = open_encoding
        if bs_kwargs is None:
            bs_kwargs = {"features": "lxml"}
        self.bs_kwargs = bs_kwargs
        self.get_text_separator = get_text_separator
[docs]    def load(self) -> List[Document]:
        """Load HTML document into document objects."""
        from bs4 import BeautifulSoup


"""Load HTML document into document objects."""
        from bs4 import BeautifulSoup
        with open(self.file_path, "r", encoding=self.open_encoding) as f:
            soup = BeautifulSoup(f, **self.bs_kwargs)
        text = soup.get_text(self.get_text_separator)
        if soup.title:
            title = str(soup.title.string)
        else:
            title = ""
        metadata: Dict[str, Union[str, None]] = {
            "source": self.file_path,
            "title": title,
        }
        return [Document(page_content=text, metadata=metadata)]


Source code for langchain.document_loaders.helpers
"""Document loader helpers."""
import concurrent.futures
from typing import List, NamedTuple, Optional, cast
[docs]class FileEncoding(NamedTuple):
    """A file encoding as the NamedTuple."""
    encoding: Optional[str]
    """The encoding of the file."""
    confidence: float
    """The confidence of the encoding."""
    language: Optional[str]
    """The language of the file."""
[docs]def detect_file_encodings(file_path: str, timeout: int = 5) -> List[FileEncoding]:
    """Try to detect the file encoding.
    Returns a list of `FileEncoding` tuples with the detected encodings ordered
    by confidence.
    Args:
        file_path: The path to the file to detect the encoding for.
        timeout: The timeout in seconds for the encoding detection.
    """
    import chardet
    def read_and_detect(file_path: str) -> List[dict]:
        with open(file_path, "rb") as f:
            rawdata = f.read()
        return cast(List[dict], chardet.detect_all(rawdata))
    with concurrent.futures.ThreadPoolExecutor() as executor:
        future = executor.submit(read_and_detect, file_path)
        try:
            encodings = future.result(timeout=timeout)
        except concurrent.futures.TimeoutError:
            raise TimeoutError(
                f"Timeout reached while detecting encoding for {file_path}"
            )
    if all(encoding["encoding"] is None for encoding in encodings):
        raise RuntimeError(f"Could not detect encoding for {file_path}")
    return [FileEncoding(**enc) for enc in encodings if enc["encoding"] is not None]


Source code for langchain.document_loaders.url
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class UnstructuredURLLoader(BaseLoader):
    """Loader that uses unstructured to load HTML files."""
    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        mode: str = "single",
        show_progress_bar: bool = False,
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        try:
            import unstructured  # noqa:F401
            from unstructured.__version__ import __version__ as __unstructured_version__
            self.__version = __unstructured_version__
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self._validate_mode(mode)
        self.mode = mode
        headers = unstructured_kwargs.pop("headers", {})
        if len(headers.keys()) != 0:
            warn_about_headers = False
            if self.__is_non_html_available():
                warn_about_headers = not self.__is_headers_available_for_non_html()
            else:
                warn_about_headers = not self.__is_headers_available_for_html()
            if warn_about_headers:
                logger.warning(
                    "You are using an old version of unstructured. "
                    "The headers parameter is ignored"
                )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.headers = headers
        self.unstructured_kwargs = unstructured_kwargs
        self.show_progress_bar = show_progress_bar


self.unstructured_kwargs = unstructured_kwargs
        self.show_progress_bar = show_progress_bar
    def _validate_mode(self, mode: str) -> None:
        _valid_modes = {"single", "elements"}
        if mode not in _valid_modes:
            raise ValueError(
                f"Got {mode} for `mode`, but should be one of `{_valid_modes}`"
            )
    def __is_headers_available_for_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 7)
    def __is_headers_available_for_non_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 13)
    def __is_non_html_available(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 12)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from unstructured.partition.auto import partition
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        if self.show_progress_bar:
            try:
                from tqdm import tqdm
            except ImportError as e:
                raise ImportError(
                    "Package tqdm must be installed if show_progress_bar=True. "
                    "Please install with 'pip install tqdm' or set "


"Please install with 'pip install tqdm' or set "
                    "show_progress_bar=False."
                ) from e
            urls = tqdm(self.urls)
        else:
            urls = self.urls
        for url in urls:
            try:
                if self.__is_non_html_available():
                    if self.__is_headers_available_for_non_html():
                        elements = partition(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition(url=url, **self.unstructured_kwargs)
                else:
                    if self.__is_headers_available_for_html():
                        elements = partition_html(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition_html(url=url, **self.unstructured_kwargs)
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                    continue
                else:
                    raise e
            if self.mode == "single":
                text = "\n\n".join([str(el) for el in elements])
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            elif self.mode == "elements":
                for element in elements:
                    metadata = element.metadata.to_dict()
                    metadata["category"] = element.category
                    docs.append(Document(page_content=str(element), metadata=metadata))
        return docs