viruPVS/langchain-docs huggingface dataset:
.execute()
        )
        request = service.files().export_media(fileId=id, mimeType="text/plain")
        fh = BytesIO()
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        try:
            while done is False:
                status, done = downloader.next_chunk()
        except HttpError as e:
            if e.resp.status == 404:
                print("File not found: {}".format(id))
            else:
                print("An error occurred: {}".format(e))
        text = fh.getvalue().decode("utf-8")
        metadata = {
            "source": f"https://docs.google.com/document/d/{id}/edit",
            "title": f"{file.get('name')}",
            "when": f"{file.get('modifiedTime')}",
        }
        return Document(page_content=text, metadata=metadata)
    def _load_documents_from_folder(
        self, folder_id: str, *, file_types: Optional[Sequence[str]] = None
    ) -> List[Document]:
        """Load documents from a folder."""
        from googleapiclient.discovery import build
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        files = self._fetch_files_recursive(service, folder_id)
        # If file types filter is provided, we'll filter by the file type.
        if file_types:
            _files = [f for f in files if f["mimeType"] in file_types]  # type: ignore
        else:
            _files = files
        returns = []
        for file in _files:
            if file["trashed"] and not self.load_trashed_files:
                continue
            elif file["mimeType"] == "application/vnd.google-apps.document":
continue
            elif file["mimeType"] == "application/vnd.google-apps.document":
                returns.append(self._load_document_from_id(file["id"]))  # type: ignore
            elif file["mimeType"] == "application/vnd.google-apps.spreadsheet":
                returns.extend(self._load_sheet_from_id(file["id"]))  # type: ignore
            elif (
                file["mimeType"] == "application/pdf"
                or self.file_loader_cls is not None
            ):
                returns.extend(self._load_file_from_id(file["id"]))  # type: ignore
            else:
                pass
        return returns
    def _fetch_files_recursive(
        self, service: Any, folder_id: str
    ) -> List[Dict[str, Union[str, List[str]]]]:
        """Fetch all files and subfolders recursively."""
        results = (
            service.files()
            .list(
                q=f"'{folder_id}' in parents",
                pageSize=1000,
                includeItemsFromAllDrives=True,
                supportsAllDrives=True,
                fields="nextPageToken, files(id, name, mimeType, parents, trashed)",
            )
            .execute()
        )
        files = results.get("files", [])
        returns = []
        for file in files:
            if file["mimeType"] == "application/vnd.google-apps.folder":
                if self.recursive:
                    returns.extend(self._fetch_files_recursive(service, file["id"]))
            else:
                returns.append(file)
        return returns
    def _load_documents_from_ids(self) -> List[Document]:
        """Load documents from a list of IDs."""
        if not self.document_ids:
            raise ValueError("document_ids must be set")
if not self.document_ids:
            raise ValueError("document_ids must be set")
        return [self._load_document_from_id(doc_id) for doc_id in self.document_ids]
    def _load_file_from_id(self, id: str) -> List[Document]:
        """Load a file from an ID."""
        from io import BytesIO
        from googleapiclient.discovery import build
        from googleapiclient.http import MediaIoBaseDownload
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        file = service.files().get(fileId=id, supportsAllDrives=True).execute()
        request = service.files().get_media(fileId=id)
        fh = BytesIO()
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        while done is False:
            status, done = downloader.next_chunk()
        if self.file_loader_cls is not None:
            fh.seek(0)
            loader = self.file_loader_cls(file=fh, **self.file_loader_kwargs)
            docs = loader.load()
            for doc in docs:
                doc.metadata["source"] = f"https://drive.google.com/file/d/{id}/view"
                if "title" not in doc.metadata:
                    doc.metadata["title"] = f"{file.get('name')}"
            return docs
        else:
            from PyPDF2 import PdfReader
            content = fh.getvalue()
            pdf_reader = PdfReader(BytesIO(content))
            return [
                Document(
                    page_content=page.extract_text(),
                    metadata={
                        "source": f"https://drive.google.com/file/d/{id}/view",
                        "title": f"{file.get('name')}",
                        "page": i,
                    },
                )
"page": i,
                    },
                )
                for i, page in enumerate(pdf_reader.pages)
            ]
    def _load_file_from_ids(self) -> List[Document]:
        """Load files from a list of IDs."""
        if not self.file_ids:
            raise ValueError("file_ids must be set")
        docs = []
        for file_id in self.file_ids:
            docs.extend(self._load_file_from_id(file_id))
        return docs
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        if self.folder_id:
            return self._load_documents_from_folder(
                self.folder_id, file_types=self.file_types
            )
        elif self.document_ids:
            return self._load_documents_from_ids()
        else:
            return self._load_file_from_ids()
Source code for langchain_community.document_loaders.git
import os
from typing import Callable, List, Optional
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class GitLoader(BaseLoader):
    """Load `Git` repository files.
    The Repository can be local on disk available at `repo_path`,
    or remote at `clone_url` that will be cloned to `repo_path`.
    Currently, supports only text files.
    Each document represents one file in the repository. The `path` points to
    the local Git repository, and the `branch` specifies the branch to load
    files from. By default, it loads from the `main` branch.
    """
[docs]    def __init__(
        self,
        repo_path: str,
        clone_url: Optional[str] = None,
        branch: Optional[str] = "main",
        file_filter: Optional[Callable[[str], bool]] = None,
    ):
        """
        Args:
            repo_path: The path to the Git repository.
            clone_url: Optional. The URL to clone the repository from.
            branch: Optional. The branch to load files from. Defaults to `main`.
            file_filter: Optional. A function that takes a file path and returns
              a boolean indicating whether to load the file. Defaults to None.
        """
        self.repo_path = repo_path
        self.clone_url = clone_url
        self.branch = branch
        self.file_filter = file_filter
[docs]    def load(self) -> List[Document]:
        try:
            from git import Blob, Repo  # type: ignore
        except ImportError as ex:
            raise ImportError(
                "Could not import git python package. "
raise ImportError(
                "Could not import git python package. "
                "Please install it with `pip install GitPython`."
            ) from ex
        if not os.path.exists(self.repo_path) and self.clone_url is None:
            raise ValueError(f"Path {self.repo_path} does not exist")
        elif self.clone_url:
            # If the repo_path already contains a git repository, verify that it's the
            # same repository as the one we're trying to clone.
            if os.path.isdir(os.path.join(self.repo_path, ".git")):
                repo = Repo(self.repo_path)
                # If the existing repository is not the same as the one we're trying to
                # clone, raise an error.
                if repo.remotes.origin.url != self.clone_url:
                    raise ValueError(
                        "A different repository is already cloned at this path."
                    )
            else:
                repo = Repo.clone_from(self.clone_url, self.repo_path)
            repo.git.checkout(self.branch)
        else:
            repo = Repo(self.repo_path)
            repo.git.checkout(self.branch)
        docs: List[Document] = []
        for item in repo.tree().traverse():
            if not isinstance(item, Blob):
                continue
            file_path = os.path.join(self.repo_path, item.path)
            ignored_files = repo.ignored([file_path])  # type: ignore
            if len(ignored_files):
                continue
            # uses filter to skip files
            if self.file_filter and not self.file_filter(file_path):
                continue
            rel_file_path = os.path.relpath(file_path, self.repo_path)
            try:
                with open(file_path, "rb") as f:
                    content = f.read()
                    file_type = os.path.splitext(item.name)[1]
content = f.read()
                    file_type = os.path.splitext(item.name)[1]
                    # loads only text files
                    try:
                        text_content = content.decode("utf-8")
                    except UnicodeDecodeError:
                        continue
                    metadata = {
                        "source": rel_file_path,
                        "file_path": rel_file_path,
                        "file_name": item.name,
                        "file_type": file_type,
                    }
                    doc = Document(page_content=text_content, metadata=metadata)
                    docs.append(doc)
            except Exception as e:
                print(f"Error reading file {file_path}: {e}")
        return docs
Source code for langchain_community.document_loaders.gitbook
from typing import Any, List, Optional
from urllib.parse import urljoin, urlparse
from langchain_core.documents import Document
from langchain_community.document_loaders.web_base import WebBaseLoader
[docs]class GitbookLoader(WebBaseLoader):
    """Load `GitBook` data.
    1. load from either a single page, or
    2. load all (relative) paths in the navbar.
    """
[docs]    def __init__(
        self,
        web_page: str,
        load_all_paths: bool = False,
        base_url: Optional[str] = None,
        content_selector: str = "main",
        continue_on_failure: bool = False,
    ):
        """Initialize with web page and whether to load all paths.
        Args:
            web_page: The web page to load or the starting point from where
                relative paths are discovered.
            load_all_paths: If set to True, all relative paths in the navbar
                are loaded instead of only `web_page`.
            base_url: If `load_all_paths` is True, the relative paths are
                appended to this base url. Defaults to `web_page`.
            content_selector: The CSS selector for the content to load.
                Defaults to "main".
            continue_on_failure: whether to continue loading the sitemap if an error
                occurs loading a url, emitting a warning instead of raising an
                exception. Setting this to True makes the loader more robust, but also
                may result in missing data. Default: False
        """
        self.base_url = base_url or web_page
        if self.base_url.endswith("/"):
            self.base_url = self.base_url[:-1]
        if load_all_paths:
self.base_url = self.base_url[:-1]
        if load_all_paths:
            # set web_path to the sitemap if we want to crawl all paths
            web_page = f"{self.base_url}/sitemap.xml"
        super().__init__(web_paths=(web_page,), continue_on_failure=continue_on_failure)
        self.load_all_paths = load_all_paths
        self.content_selector = content_selector
[docs]    def load(self) -> List[Document]:
        """Fetch text from one single GitBook page."""
        if self.load_all_paths:
            soup_info = self.scrape()
            relative_paths = self._get_paths(soup_info)
            urls = [urljoin(self.base_url, path) for path in relative_paths]
            soup_infos = self.scrape_all(urls)
            _documents = [
                self._get_document(soup_info, url)
                for soup_info, url in zip(soup_infos, urls)
            ]
        else:
            soup_info = self.scrape()
            _documents = [self._get_document(soup_info, self.web_path)]
        documents = [d for d in _documents if d]
        return documents
    def _get_document(
        self, soup: Any, custom_url: Optional[str] = None
    ) -> Optional[Document]:
        """Fetch content from page and return Document."""
        page_content_raw = soup.find(self.content_selector)
        if not page_content_raw:
            return None
        content = page_content_raw.get_text(separator="\n").strip()
        title_if_exists = page_content_raw.find("h1")
        title = title_if_exists.text if title_if_exists else ""
        metadata = {"source": custom_url or self.web_path, "title": title}
        return Document(page_content=content, metadata=metadata)
Source code for langchain_community.document_loaders.odt
from typing import Any, List
from langchain_community.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredODTLoader(UnstructuredFileLoader):
    """Load `OpenOffice ODT` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain_community.document_loaders import UnstructuredODTLoader
    loader = UnstructuredODTLoader(
        "example.odt", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-odt
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """
        Args:
            file_path: The path to the file to load.
            mode: The mode to use when loading the file. Can be one of "single",
                "multi", or "all". Default is "single".
            **unstructured_kwargs: Any kwargs to pass to the unstructured.
        """
        validate_unstructured_version(min_unstructured_version="0.6.3")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
Source code for langchain_community.document_loaders.gcs_file
import os
import tempfile
from typing import Callable, List, Optional
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
from langchain_community.utilities.vertexai import get_client_info
[docs]class GCSFileLoader(BaseLoader):
    """Load from GCS file."""
[docs]    def __init__(
        self,
        project_name: str,
        bucket: str,
        blob: str,
        loader_func: Optional[Callable[[str], BaseLoader]] = None,
    ):
        """Initialize with bucket and key name.
        Args:
            project_name: The name of the project to load
            bucket: The name of the GCS bucket.
            blob: The name of the GCS blob to load.
            loader_func: A loader function that instantiates a loader based on a
                file_path argument. If nothing is provided, the
                UnstructuredFileLoader is used.
        Examples:
            To use an alternative PDF loader:
            >> from from langchain_community.document_loaders import PyPDFLoader
            >> loader = GCSFileLoader(..., loader_func=PyPDFLoader)
            To use UnstructuredFileLoader with additional arguments:
            >> loader = GCSFileLoader(...,
            >>      loader_func=lambda x: UnstructuredFileLoader(x, mode="elements"))
        """
        self.bucket = bucket
        self.blob = blob
        self.project_name = project_name
        def default_loader_func(file_path: str) -> BaseLoader:
            return UnstructuredFileLoader(file_path)
        self._loader_func = loader_func if loader_func else default_loader_func
self._loader_func = loader_func if loader_func else default_loader_func
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ImportError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        # Initialise a client
        storage_client = storage.Client(
            self.project_name, client_info=get_client_info("google-cloud-storage")
        )
        # Create a bucket object for our bucket
        bucket = storage_client.get_bucket(self.bucket)
        # Create a blob object from the filepath
        blob = bucket.blob(self.blob)
        # retrieve custom metadata associated with the blob
        metadata = bucket.get_blob(self.blob).metadata
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.blob}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            # Download the file to a destination
            blob.download_to_filename(file_path)
            loader = self._loader_func(file_path)
            docs = loader.load()
            for doc in docs:
                if "source" in doc.metadata:
                    doc.metadata["source"] = f"gs://{self.bucket}/{self.blob}"
                if metadata:
                    doc.metadata.update(metadata)
            return docs
Source code for langchain_community.document_loaders.diffbot
import logging
from typing import Any, List
import requests
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class DiffbotLoader(BaseLoader):
    """Load `Diffbot` json file."""
[docs]    def __init__(
        self, api_token: str, urls: List[str], continue_on_failure: bool = True
    ):
        """Initialize with API token, ids, and key.
        Args:
            api_token: Diffbot API token.
            urls: List of URLs to load.
            continue_on_failure: Whether to continue loading other URLs if one fails.
               Defaults to True.
        """
        self.api_token = api_token
        self.urls = urls
        self.continue_on_failure = continue_on_failure
    def _diffbot_api_url(self, diffbot_api: str) -> str:
        return f"https://api.diffbot.com/v3/{diffbot_api}"
    def _get_diffbot_data(self, url: str) -> Any:
        """Get Diffbot file from Diffbot REST API."""
        # TODO: Add support for other Diffbot APIs
        diffbot_url = self._diffbot_api_url("article")
        params = {
            "token": self.api_token,
            "url": url,
        }
        response = requests.get(diffbot_url, params=params, timeout=10)
        # TODO: handle non-ok errors
        return response.json() if response.ok else {}
[docs]    def load(self) -> List[Document]:
        """Extract text from Diffbot on all the URLs and return Documents"""
        docs: List[Document] = list()
        for url in self.urls:
docs: List[Document] = list()
        for url in self.urls:
            try:
                data = self._get_diffbot_data(url)
                text = data["objects"][0]["text"] if "objects" in data else ""
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                else:
                    raise e
        return docs
Source code for langchain_community.document_loaders.rocksetdb
from typing import Any, Callable, Iterator, List, Optional, Tuple
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]def default_joiner(docs: List[Tuple[str, Any]]) -> str:
    """Default joiner for content columns."""
    return "\n".join([doc[1] for doc in docs])
[docs]class ColumnNotFoundError(Exception):
    """Column not found error."""
    def __init__(self, missing_key: str, query: str):
        super().__init__(f'Column "{missing_key}" not selected in query:\n{query}')
[docs]class RocksetLoader(BaseLoader):
    """Load from a `Rockset` database.
    To use, you should have the `rockset` python package installed.
    Example:
        .. code-block:: python
            # This code will load 3 records from the "langchain_demo"
            # collection as Documents, with the `text` column used as
            # the content
            from langchain_community.document_loaders import RocksetLoader
            from rockset import RocksetClient, Regions, models
            loader = RocksetLoader(
                RocksetClient(Regions.usw2a1, "<api key>"),
                models.QueryRequestSql(
                    query="select * from langchain_demo limit 3"
                ),
                ["text"]
            )
        )
    """
[docs]    def __init__(
        self,
        client: Any,
        query: Any,
        content_keys: List[str],
        metadata_keys: Optional[List[str]] = None,
        content_columns_joiner: Callable[[List[Tuple[str, Any]]], str] = default_joiner,
):
        """Initialize with Rockset client.
        Args:
            client: Rockset client object.
            query: Rockset query object.
            content_keys: The collection columns to be written into the `page_content`
                of the Documents.
            metadata_keys: The collection columns to be written into the `metadata` of
                the Documents. By default, this is all the keys in the document.
            content_columns_joiner: Method that joins content_keys and its values into a
                string. It's method that takes in a List[Tuple[str, Any]]],
                representing a list of tuples of (column name, column value).
                By default, this is a method that joins each column value with a new
                line. This method is only relevant if there are multiple content_keys.
        """
        try:
            from rockset import QueryPaginator, RocksetClient
            from rockset.models import QueryRequestSql
        except ImportError:
            raise ImportError(
                "Could not import rockset client python package. "
                "Please install it with `pip install rockset`."
            )
        if not isinstance(client, RocksetClient):
            raise ValueError(
                f"client should be an instance of rockset.RocksetClient, "
                f"got {type(client)}"
            )
        if not isinstance(query, QueryRequestSql):
            raise ValueError(
                f"query should be an instance of rockset.model.QueryRequestSql, "
                f"got {type(query)}"
            )
        self.client = client
        self.query = query
        self.content_keys = content_keys
        self.content_columns_joiner = content_columns_joiner
        self.metadata_keys = metadata_keys
        self.paginator = QueryPaginator
        self.request_model = QueryRequestSql
        try: