viruPVS/langchain-docs huggingface dataset:
attachments.append(href)
        return attachments
    def _download_attachments(self, attachments: List[str]) -> None:
        """Download all attachments.
        Args:
            attachments: List of attachments.
        """
        # Make sure the folder exists
        Path(self.folder_path).mkdir(parents=True, exist_ok=True)
        # Download all attachments
        for attachment in attachments:
            self.download(attachment)
    def _load_documents(self) -> List[Document]:
        """Load all documents in the folder.
        Returns:
            List of documents.
        """
        # Create the document loader
        loader = DirectoryLoader(
            path=self.folder_path,
            glob="*.pdf",
            loader_cls=PyPDFLoader,  # type: ignore
        )
        # Load the documents
        documents = loader.load()
        # Return all documents
        return documents
    def _get_paths(self, soup: Any) -> List[str]:
        """Get all relative paths in the navbar."""
        relative_paths = []
        course_menu = soup.find("ul", {"class": "courseMenu"})
        if course_menu is None:
            raise ValueError("No course menu found.")
        for link in course_menu.find_all("a"):
            href = link.get("href")
            if href is not None and href.startswith("/"):
                relative_paths.append(href)
        return relative_paths
[docs]    def download(self, path: str) -> None:
        """Download a file from an url.
        Args:
            path: Path to the file.
        """
        # Get the file content
        response = self.session.get(self.base_url + path, allow_redirects=True)
        # Get the filename
        filename = self.parse_filename(response.url)
        # Write the file to disk
filename = self.parse_filename(response.url)
        # Write the file to disk
        with open(Path(self.folder_path) / filename, "wb") as f:
            f.write(response.content)
[docs]    def parse_filename(self, url: str) -> str:
        """Parse the filename from an url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        """
        if (url_path := Path(url)) and url_path.suffix == ".pdf":
            return url_path.name
        else:
            return self._parse_filename_from_url(url)
    def _parse_filename_from_url(self, url: str) -> str:
        """Parse the filename from an url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        Raises:
            ValueError: If the filename could not be parsed.
        """
        filename_matches = re.search(r"filename%2A%3DUTF-8%27%27(.+)", url)
        if filename_matches:
            filename = filename_matches.group(1)
        else:
            raise ValueError(f"Could not parse filename from {url}")
        if ".pdf" not in filename:
            raise ValueError(f"Incorrect file type: {filename}")
        filename = filename.split(".pdf")[0] + ".pdf"
        filename = unquote(filename)
        filename = filename.replace("%20", " ")
        return filename
if __name__ == "__main__":
    loader = BlackboardLoader(
        "https://<YOUR BLACKBOARD URL"
        " HERE>/webapps/blackboard/content/listContent.jsp?course_id=_<YOUR COURSE ID"
Source code for langchain_community.document_loaders.base_o365
"""Base class for all loaders that uses O365 Package"""
from __future__ import annotations
import logging
import os
import tempfile
from abc import abstractmethod
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, Dict, Iterable, List, Sequence, Union
from langchain_core.pydantic_v1 import (
    BaseModel,
    BaseSettings,
    Field,
    FilePath,
    SecretStr,
)
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.blob_loaders.file_system import (
    FileSystemBlobLoader,
)
from langchain_community.document_loaders.blob_loaders.schema import Blob
if TYPE_CHECKING:
    from O365 import Account
    from O365.drive import Drive, Folder
logger = logging.getLogger(__name__)
CHUNK_SIZE = 1024 * 1024 * 5
class _O365Settings(BaseSettings):
    client_id: str = Field(..., env="O365_CLIENT_ID")
    client_secret: SecretStr = Field(..., env="O365_CLIENT_SECRET")
    class Config:
        env_prefix = ""
        case_sentive = False
        env_file = ".env"
class _O365TokenStorage(BaseSettings):
    token_path: FilePath = Path.home() / ".credentials" / "o365_token.txt"
class _FileType(str, Enum):
    DOC = "doc"
    DOCX = "docx"
    PDF = "pdf"
[docs]def fetch_mime_types(file_types: Sequence[_FileType]) -> Dict[str, str]:
    """Fetch the mime types for the specified file types."""
    mime_types_mapping = {}
    for file_type in file_types:
        if file_type.value == "doc":
for file_type in file_types:
        if file_type.value == "doc":
            mime_types_mapping[file_type.value] = "application/msword"
        elif file_type.value == "docx":
            mime_types_mapping[
                file_type.value
            ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"  # noqa: E501
        elif file_type.value == "pdf":
            mime_types_mapping[file_type.value] = "application/pdf"
    return mime_types_mapping
[docs]class O365BaseLoader(BaseLoader, BaseModel):
    """Base class for all loaders that uses O365 Package"""
    settings: _O365Settings = Field(default_factory=_O365Settings)
    """Settings for the Office365 API client."""
    auth_with_token: bool = False
    """Whether to authenticate with a token or not. Defaults to False."""
    chunk_size: Union[int, str] = CHUNK_SIZE
    """Number of bytes to retrieve from each api call to the server. int or 'auto'."""
    @property
    @abstractmethod
    def _file_types(self) -> Sequence[_FileType]:
        """Return supported file types."""
    @property
    def _fetch_mime_types(self) -> Dict[str, str]:
        """Return a dict of supported file types to corresponding mime types."""
        return fetch_mime_types(self._file_types)
    @property
    @abstractmethod
    def _scopes(self) -> List[str]:
        """Return required scopes."""
    def _load_from_folder(self, folder: Folder) -> Iterable[Blob]:
        """Lazily load all files from a specified folder of the configured MIME type.
        Args:
            folder: The Folder instance from which the files are to be loaded. This
                Folder instance should represent a directory in a file system where the
Folder instance should represent a directory in a file system where the
                files are stored.
        Yields:
            An iterator that yields Blob instances, which are binary representations of
                the files loaded from the folder.
        """
        file_mime_types = self._fetch_mime_types
        items = folder.get_items()
        with tempfile.TemporaryDirectory() as temp_dir:
            os.makedirs(os.path.dirname(temp_dir), exist_ok=True)
            for file in items:
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        file.download(to_path=temp_dir, chunk_size=self.chunk_size)
            loader = FileSystemBlobLoader(path=temp_dir)
            yield from loader.yield_blobs()
    def _load_from_object_ids(
        self, drive: Drive, object_ids: List[str]
    ) -> Iterable[Blob]:
        """Lazily load files specified by their object_ids from a drive.
        Load files into the system as binary large objects (Blobs) and return Iterable.
        Args:
            drive: The Drive instance from which the files are to be loaded. This Drive
                instance should represent a cloud storage service or similar storage
                system where the files are stored.
            object_ids: A list of object_id strings. Each object_id represents a unique
                identifier for a file in the drive.
        Yields:
            An iterator that yields Blob instances, which are binary representations of
            the files loaded from the drive using the specified object_ids.
        """
        file_mime_types = self._fetch_mime_types
        with tempfile.TemporaryDirectory() as temp_dir:
            for object_id in object_ids:
                file = drive.get_item(object_id)
                if not file:
                    logging.warning(
                        "There isn't a file with"
logging.warning(
                        "There isn't a file with"
                        f"object_id {object_id} in drive {drive}."
                    )
                    continue
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        file.download(to_path=temp_dir, chunk_size=self.chunk_size)
            loader = FileSystemBlobLoader(path=temp_dir)
            yield from loader.yield_blobs()
    def _auth(self) -> Account:
        """Authenticates the OneDrive API client
        Returns:
            The authenticated Account object.
        """
        try:
            from O365 import Account, FileSystemTokenBackend
        except ImportError:
            raise ImportError(
                "O365 package not found, please install it with `pip install o365`"
            )
        if self.auth_with_token:
            token_storage = _O365TokenStorage()
            token_path = token_storage.token_path
            token_backend = FileSystemTokenBackend(
                token_path=token_path.parent, token_filename=token_path.name
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=self._scopes,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
        else:
            token_backend = FileSystemTokenBackend(
                token_path=Path.home() / ".credentials"
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=self._scopes,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
            # make the auth
            account.authenticate()
        return account
Source code for langchain_community.document_loaders.merge
from typing import Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class MergedDataLoader(BaseLoader):
    """Merge documents from a list of loaders"""
[docs]    def __init__(self, loaders: List):
        """Initialize with a list of loaders"""
        self.loaders = loaders
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load docs from each individual loader."""
        for loader in self.loaders:
            # Check if lazy_load is implemented
            try:
                data = loader.lazy_load()
            except NotImplementedError:
                data = loader.load()
            for document in data:
                yield document
[docs]    def load(self) -> List[Document]:
        """Load docs."""
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.chromium
import asyncio
import logging
from typing import Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class AsyncChromiumLoader(BaseLoader):
    """Scrape HTML pages from URLs using a
    headless instance of the Chromium."""
[docs]    def __init__(
        self,
        urls: List[str],
    ):
        """
        Initialize the loader with a list of URL paths.
        Args:
            urls (List[str]): A list of URLs to scrape content from.
        Raises:
            ImportError: If the required 'playwright' package is not installed.
        """
        self.urls = urls
        try:
            import playwright  # noqa: F401
        except ImportError:
            raise ImportError(
                "playwright is required for AsyncChromiumLoader. "
                "Please install it with `pip install playwright`."
            )
[docs]    async def ascrape_playwright(self, url: str) -> str:
        """
        Asynchronously scrape the content of a given URL using Playwright's async API.
        Args:
            url (str): The URL to scrape.
        Returns:
            str: The scraped HTML content or an error message if an exception occurs.
        """
        from playwright.async_api import async_playwright
        logger.info("Starting scraping...")
        results = ""
        async with async_playwright() as p:
            browser = await p.chromium.launch(headless=True)
            try:
                page = await browser.new_page()
                await page.goto(url)
                results = await page.content()  # Simply get the HTML content
                logger.info("Content scraped")
logger.info("Content scraped")
            except Exception as e:
                results = f"Error: {e}"
            await browser.close()
        return results
[docs]    def lazy_load(self) -> Iterator[Document]:
        """
        Lazily load text content from the provided URLs.
        This method yields Documents one at a time as they're scraped,
        instead of waiting to scrape all URLs before returning.
        Yields:
            Document: The scraped content encapsulated within a Document object.
        """
        for url in self.urls:
            html_content = asyncio.run(self.ascrape_playwright(url))
            metadata = {"source": url}
            yield Document(page_content=html_content, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """
        Load and return all Documents from the provided URLs.
        Returns:
            List[Document]: A list of Document objects
            containing the scraped content from each URL.
        """
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.rtf
"""Loads rich text files."""
from typing import Any, List
from langchain_community.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredRTFLoader(UnstructuredFileLoader):
    """Load `RTF` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain_community.document_loaders import UnstructuredRTFLoader
    loader = UnstructuredRTFLoader(
        "example.rtf", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-rtf
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """
        Initialize with a file path.
        Args:
            file_path: The path to the file to load.
            mode: The mode to use for partitioning. See unstructured for details.
                Defaults to "single".
            **unstructured_kwargs: Additional keyword arguments to pass
                to unstructured.
        """
        min_unstructured_version = "0.5.12"
        if not satisfies_min_unstructured_version(min_unstructured_version):
if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning rtf files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.rtf import partition_rtf
        return partition_rtf(filename=self.file_path, **self.unstructured_kwargs)
Source code for langchain_community.document_loaders.sharepoint
"""Loader that loads data from Sharepoint Document Library"""
from __future__ import annotations
from typing import Iterator, List, Optional, Sequence
from langchain_core.documents import Document
from langchain_core.pydantic_v1 import Field
from langchain_community.document_loaders.base_o365 import (
    O365BaseLoader,
    _FileType,
)
from langchain_community.document_loaders.parsers.registry import get_parser
[docs]class SharePointLoader(O365BaseLoader):
    """Load  from `SharePoint`."""
    document_library_id: str = Field(...)
    """ The ID of the SharePoint document library to load data from."""
    folder_path: Optional[str] = None
    """ The path to the folder to load data from."""
    object_ids: Optional[List[str]] = None
    """ The IDs of the objects to load data from."""
    @property
    def _file_types(self) -> Sequence[_FileType]:
        """Return supported file types."""
        return _FileType.DOC, _FileType.DOCX, _FileType.PDF
    @property
    def _scopes(self) -> List[str]:
        """Return required scopes."""
        return ["sharepoint", "basic"]
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load documents lazily. Use this when working at a large scale."""
        try:
            from O365.drive import Drive, Folder
        except ImportError:
            raise ImportError(
                "O365 package not found, please install it with `pip install o365`"
            )
        drive = self._auth().storage().get_drive(self.document_library_id)
        if not isinstance(drive, Drive):
            raise ValueError(f"There isn't a Drive with id {self.document_library_id}.")
raise ValueError(f"There isn't a Drive with id {self.document_library_id}.")
        blob_parser = get_parser("default")
        if self.folder_path:
            target_folder = drive.get_item_by_path(self.folder_path)
            if not isinstance(target_folder, Folder):
                raise ValueError(f"There isn't a folder with path {self.folder_path}.")
            for blob in self._load_from_folder(target_folder):
                yield from blob_parser.lazy_parse(blob)
        if self.object_ids:
            for blob in self._load_from_object_ids(drive, self.object_ids):
                yield from blob_parser.lazy_parse(blob)
[docs]    def load(self) -> List[Document]:
        """Load all documents."""
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.tencent_cos_directory
from typing import Any, Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.tencent_cos_file import TencentCOSFileLoader
[docs]class TencentCOSDirectoryLoader(BaseLoader):
    """Load from `Tencent Cloud COS` directory."""
[docs]    def __init__(self, conf: Any, bucket: str, prefix: str = ""):
        """Initialize with COS config, bucket and prefix.
        :param conf(CosConfig): COS config.
        :param bucket(str): COS bucket.
        :param prefix(str): prefix.
        """
        self.conf = conf
        self.bucket = bucket
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load documents."""
        try:
            from qcloud_cos import CosS3Client
        except ImportError:
            raise ImportError(
                "Could not import cos-python-sdk-v5 python package. "
                "Please install it with `pip install cos-python-sdk-v5`."
            )
        client = CosS3Client(self.conf)
        contents = []
        marker = ""
        while True:
            response = client.list_objects(
                Bucket=self.bucket, Prefix=self.prefix, Marker=marker, MaxKeys=1000
            )
            if "Contents" in response:
                contents.extend(response["Contents"])
            if response["IsTruncated"] == "false":
                break
            marker = response["NextMarker"]
        for content in contents:
            if content["Key"].endswith("/"):
                continue
Source code for langchain_community.document_loaders.imsdb
from typing import List
from langchain_core.documents import Document
from langchain_community.document_loaders.web_base import WebBaseLoader
[docs]class IMSDbLoader(WebBaseLoader):
    """Load `IMSDb` webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        text = soup.select_one("td[class='scrtext']").text
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
Source code for langchain_community.document_loaders.psychic
from typing import List, Optional
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class PsychicLoader(BaseLoader):
    """Load from `Psychic.dev`."""
[docs]    def __init__(
        self, api_key: str, account_id: str, connector_id: Optional[str] = None
    ):
        """Initialize with API key, connector id, and account id.
        Args:
            api_key: The Psychic API key.
            account_id: The Psychic account id.
            connector_id: The Psychic connector id.
        """
        try:
            from psychicapi import ConnectorId, Psychic  # noqa: F401
        except ImportError:
            raise ImportError(
                "`psychicapi` package not found, please run `pip install psychicapi`"
            )
        self.psychic = Psychic(secret_key=api_key)
        self.connector_id = ConnectorId(connector_id)
        self.account_id = account_id
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        psychic_docs = self.psychic.get_documents(
            connector_id=self.connector_id, account_id=self.account_id
        )
        return [
            Document(
                page_content=doc["content"],
                metadata={"title": doc["title"], "source": doc["uri"]},
            )
            for doc in psychic_docs.documents
        ]
Source code for langchain_community.document_loaders.dropbox
# Prerequisites:
# 1. Create a Dropbox app.
# 2. Give the app these scope permissions: `files.metadata.read`
#    and `files.content.read`.
# 3. Generate access token: https://www.dropbox.com/developers/apps/create.
# 4. `pip install dropbox` (requires `pip install unstructured[pdf]` for PDF filetype).
import os
import tempfile
from pathlib import Path
from typing import Any, Dict, List, Optional
from langchain_core.documents import Document
from langchain_core.pydantic_v1 import BaseModel, root_validator
from langchain_community.document_loaders.base import BaseLoader
[docs]class DropboxLoader(BaseLoader, BaseModel):
    """Load files from `Dropbox`.
    In addition to common files such as text and PDF files, it also supports
    *Dropbox Paper* files.
    """
    dropbox_access_token: str
    """Dropbox access token."""
    dropbox_folder_path: Optional[str] = None
    """The folder path to load from."""
    dropbox_file_paths: Optional[List[str]] = None
    """The file paths to load from."""
    recursive: bool = False
    """Flag to indicate whether to load files recursively from subfolders."""
    @root_validator
    def validate_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that either folder_path or file_paths is set, but not both."""
        if (
            values.get("dropbox_folder_path") is not None
            and values.get("dropbox_file_paths") is not None
        ):
            raise ValueError("Cannot specify both folder_path and file_paths")
        if values.get("dropbox_folder_path") is None and not values.get(