viruPVS/langchain-docs huggingface dataset:
if values.get("dropbox_folder_path") is None and not values.get(
            "dropbox_file_paths"
        ):
            raise ValueError("Must specify either folder_path or file_paths")
        return values
    def _create_dropbox_client(self) -> Any:
        """Create a Dropbox client."""
        try:
            from dropbox import Dropbox, exceptions
        except ImportError:
            raise ImportError("You must run " "`pip install dropbox")
        try:
            dbx = Dropbox(self.dropbox_access_token)
            dbx.users_get_current_account()
        except exceptions.AuthError as ex:
            raise ValueError(
                "Invalid Dropbox access token. Please verify your token and try again."
            ) from ex
        return dbx
    def _load_documents_from_folder(self, folder_path: str) -> List[Document]:
        """Load documents from a Dropbox folder."""
        dbx = self._create_dropbox_client()
        try:
            from dropbox import exceptions
            from dropbox.files import FileMetadata
        except ImportError:
            raise ImportError("You must run " "`pip install dropbox")
        try:
            results = dbx.files_list_folder(folder_path, recursive=self.recursive)
        except exceptions.ApiError as ex:
            raise ValueError(
                f"Could not list files in the folder: {folder_path}. "
                "Please verify the folder path and try again."
            ) from ex
        files = [entry for entry in results.entries if isinstance(entry, FileMetadata)]
        documents = [
            doc
            for doc in (self._load_file_from_path(file.path_display) for file in files)
            if doc is not None
        ]
        return documents
    def _load_file_from_path(self, file_path: str) -> Optional[Document]:
def _load_file_from_path(self, file_path: str) -> Optional[Document]:
        """Load a file from a Dropbox path."""
        dbx = self._create_dropbox_client()
        try:
            from dropbox import exceptions
        except ImportError:
            raise ImportError("You must run " "`pip install dropbox")
        try:
            file_metadata = dbx.files_get_metadata(file_path)
            if file_metadata.is_downloadable:
                _, response = dbx.files_download(file_path)
            # Some types such as Paper, need to be exported.
            elif file_metadata.export_info:
                _, response = dbx.files_export(file_path, "markdown")
        except exceptions.ApiError as ex:
            raise ValueError(
                f"Could not load file: {file_path}. Please verify the file path"
                "and try again."
            ) from ex
        try:
            text = response.content.decode("utf-8")
        except UnicodeDecodeError:
            file_extension = os.path.splitext(file_path)[1].lower()
            if file_extension == ".pdf":
                print(f"File {file_path} type detected as .pdf")
                from langchain_community.document_loaders import UnstructuredPDFLoader
                # Download it to a temporary file.
                temp_dir = tempfile.TemporaryDirectory()
                temp_pdf = Path(temp_dir.name) / "tmp.pdf"
                with open(temp_pdf, mode="wb") as f:
                    f.write(response.content)
                try:
                    loader = UnstructuredPDFLoader(str(temp_pdf))
                    docs = loader.load()
                    if docs:
                        return docs[0]
                except Exception as pdf_ex:
                    print(f"Error while trying to parse PDF {file_path}: {pdf_ex}")
                    return None
            else:
return None
            else:
                print(
                    f"File {file_path} could not be decoded as pdf or text. Skipping."
                )
            return None
        metadata = {
            "source": f"dropbox://{file_path}",
            "title": os.path.basename(file_path),
        }
        return Document(page_content=text, metadata=metadata)
    def _load_documents_from_paths(self) -> List[Document]:
        """Load documents from a list of Dropbox file paths."""
        if not self.dropbox_file_paths:
            raise ValueError("file_paths must be set")
        return [
            doc
            for doc in (
                self._load_file_from_path(file_path)
                for file_path in self.dropbox_file_paths
            )
            if doc is not None
        ]
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        if self.dropbox_folder_path is not None:
            return self._load_documents_from_folder(self.dropbox_folder_path)
        else:
            return self._load_documents_from_paths()
Source code for langchain_community.document_loaders.python
import tokenize
from langchain_community.document_loaders.text import TextLoader
[docs]class PythonLoader(TextLoader):
    """Load `Python` files, respecting any non-default encoding if specified."""
[docs]    def __init__(self, file_path: str):
        """Initialize with a file path.
        Args:
            file_path: The path to the file to load.
        """
        with open(file_path, "rb") as f:
            encoding, _ = tokenize.detect_encoding(f.readline)
        super().__init__(file_path=file_path, encoding=encoding)
Source code for langchain_community.document_loaders.base
"""Abstract interface for document loader implementations."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Iterator, List, Optional
from langchain_core.documents import Document
from langchain_community.document_loaders.blob_loaders import Blob
if TYPE_CHECKING:
    from langchain.text_splitter import TextSplitter
[docs]class BaseLoader(ABC):
    """Interface for Document Loader.
    Implementations should implement the lazy-loading method using generators
    to avoid loading all Documents into memory at once.
    The `load` method will remain as is for backwards compatibility, but its
    implementation should be just `list(self.lazy_load())`.
    """
    # Sub-classes should implement this method
    # as return list(self.lazy_load()).
    # This method returns a List which is materialized in memory.
[docs]    @abstractmethod
    def load(self) -> List[Document]:
        """Load data into Document objects."""
[docs]    def load_and_split(
        self, text_splitter: Optional[TextSplitter] = None
    ) -> List[Document]:
        """Load Documents and split into chunks. Chunks are returned as Documents.
        Args:
            text_splitter: TextSplitter instance to use for splitting documents.
              Defaults to RecursiveCharacterTextSplitter.
        Returns:
            List of Documents.
        """
        from langchain.text_splitter import RecursiveCharacterTextSplitter
        if text_splitter is None:
            _text_splitter: TextSplitter = RecursiveCharacterTextSplitter()
        else:
            _text_splitter = text_splitter
        docs = self.load()
        return _text_splitter.split_documents(docs)
docs = self.load()
        return _text_splitter.split_documents(docs)
    # Attention: This method will be upgraded into an abstractmethod once it's
    #            implemented in all the existing subclasses.
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """A lazy loader for Documents."""
        raise NotImplementedError(
            f"{self.__class__.__name__} does not implement lazy_load()"
        )
[docs]class BaseBlobParser(ABC):
    """Abstract interface for blob parsers.
    A blob parser provides a way to parse raw data stored in a blob into one
    or more documents.
    The parser can be composed with blob loaders, making it easy to reuse
    a parser independent of how the blob was originally loaded.
    """
[docs]    @abstractmethod
    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        """Lazy parsing interface.
        Subclasses are required to implement this method.
        Args:
            blob: Blob instance
        Returns:
            Generator of documents
        """
[docs]    def parse(self, blob: Blob) -> List[Document]:
        """Eagerly parse the blob into a document or documents.
        This is a convenience method for interactive development environment.
        Production applications should favor the lazy_parse method instead.
        Subclasses should generally not over-ride this parse method.
        Args:
            blob: Blob instance
        Returns:
            List of documents
        """
        return list(self.lazy_parse(blob))
Source code for langchain_community.document_loaders.cube_semantic
import json
import logging
import time
from typing import List
import requests
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class CubeSemanticLoader(BaseLoader):
    """Load `Cube semantic layer` metadata.
    Args:
        cube_api_url: REST API endpoint.
            Use the REST API of your Cube's deployment.
            Please find out more information here:
            https://cube.dev/docs/http-api/rest#configuration-base-path
        cube_api_token: Cube API token.
            Authentication tokens are generated based on your Cube's API secret.
            Please find out more information here:
            https://cube.dev/docs/security#generating-json-web-tokens-jwt
        load_dimension_values: Whether to load dimension values for every string
            dimension or not.
        dimension_values_limit: Maximum number of dimension values to load.
        dimension_values_max_retries: Maximum number of retries to load dimension
            values.
        dimension_values_retry_delay: Delay between retries to load dimension values.
    """
[docs]    def __init__(
        self,
        cube_api_url: str,
        cube_api_token: str,
        load_dimension_values: bool = True,
        dimension_values_limit: int = 10_000,
        dimension_values_max_retries: int = 10,
        dimension_values_retry_delay: int = 3,
    ):
        self.cube_api_url = cube_api_url
        self.cube_api_token = cube_api_token
        self.load_dimension_values = load_dimension_values
        self.dimension_values_limit = dimension_values_limit
        self.dimension_values_max_retries = dimension_values_max_retries
        self.dimension_values_retry_delay = dimension_values_retry_delay
self.dimension_values_retry_delay = dimension_values_retry_delay
    def _get_dimension_values(self, dimension_name: str) -> List[str]:
        """Makes a call to Cube's REST API load endpoint to retrieve
        values for dimensions.
        These values can be used to achieve a more accurate filtering.
        """
        logger.info("Loading dimension values for: {dimension_name}...")
        headers = {
            "Content-Type": "application/json",
            "Authorization": self.cube_api_token,
        }
        query = {
            "query": {
                "dimensions": [dimension_name],
                "limit": self.dimension_values_limit,
            }
        }
        retries = 0
        while retries < self.dimension_values_max_retries:
            response = requests.request(
                "POST",
                f"{self.cube_api_url}/load",
                headers=headers,
                data=json.dumps(query),
            )
            if response.status_code == 200:
                response_data = response.json()
                if (
                    "error" in response_data
                    and response_data["error"] == "Continue wait"
                ):
                    logger.info("Retrying...")
                    retries += 1
                    time.sleep(self.dimension_values_retry_delay)
                    continue
                else:
                    dimension_values = [
                        item[dimension_name] for item in response_data["data"]
                    ]
                    return dimension_values
            else:
                logger.error("Request failed with status code:", response.status_code)
                break
        if retries == self.dimension_values_max_retries:
            logger.info("Maximum retries reached.")
        return []
[docs]    def load(self) -> List[Document]:
        """Makes a call to Cube's REST API metadata endpoint.
        Returns:
"""Makes a call to Cube's REST API metadata endpoint.
        Returns:
            A list of documents with attributes:
                - page_content=column_title + column_description
                - metadata
                    - table_name
                    - column_name
                    - column_data_type
                    - column_member_type
                    - column_title
                    - column_description
                    - column_values
                    - cube_data_obj_type
        """
        headers = {
            "Content-Type": "application/json",
            "Authorization": self.cube_api_token,
        }
        logger.info(f"Loading metadata from {self.cube_api_url}...")
        response = requests.get(f"{self.cube_api_url}/meta", headers=headers)
        response.raise_for_status()
        raw_meta_json = response.json()
        cube_data_objects = raw_meta_json.get("cubes", [])
        logger.info(f"Found {len(cube_data_objects)} cube data objects in metadata.")
        if not cube_data_objects:
            raise ValueError("No cubes found in metadata.")
        docs = []
        for cube_data_obj in cube_data_objects:
            cube_data_obj_name = cube_data_obj.get("name")
            cube_data_obj_type = cube_data_obj.get("type")
            cube_data_obj_is_public = cube_data_obj.get("public")
            measures = cube_data_obj.get("measures", [])
            dimensions = cube_data_obj.get("dimensions", [])
            logger.info(f"Processing {cube_data_obj_name}...")
            if not cube_data_obj_is_public:
                logger.info(f"Skipping {cube_data_obj_name} because it is not public.")
                continue
            for item in measures + dimensions:
                column_member_type = "measure" if item in measures else "dimension"
                dimension_values = []
                item_name = str(item.get("name"))
dimension_values = []
                item_name = str(item.get("name"))
                item_type = str(item.get("type"))
                if (
                    self.load_dimension_values
                    and column_member_type == "dimension"
                    and item_type == "string"
                ):
                    dimension_values = self._get_dimension_values(item_name)
                metadata = dict(
                    table_name=str(cube_data_obj_name),
                    column_name=item_name,
                    column_data_type=item_type,
                    column_title=str(item.get("title")),
                    column_description=str(item.get("description")),
                    column_member_type=column_member_type,
                    column_values=dimension_values,
                    cube_data_obj_type=cube_data_obj_type,
                )
                page_content = f"{str(item.get('title'))}, "
                page_content += f"{str(item.get('description'))}"
                docs.append(Document(page_content=page_content, metadata=metadata))
        return docs
Source code for langchain_community.document_loaders.googledrive
# Prerequisites:
# 1. Create a Google Cloud project
# 2. Enable the Google Drive API:
#   https://console.cloud.google.com/flows/enableapi?apiid=drive.googleapis.com
# 3. Authorize credentials for desktop app:
#   https://developers.google.com/drive/api/quickstart/python#authorize_credentials_for_a_desktop_application # noqa: E501
# 4. For service accounts visit
#   https://cloud.google.com/iam/docs/service-accounts-create
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Union
from langchain_core.documents import Document
from langchain_core.pydantic_v1 import BaseModel, root_validator, validator
from langchain_community.document_loaders.base import BaseLoader
SCOPES = ["https://www.googleapis.com/auth/drive.readonly"]
[docs]class GoogleDriveLoader(BaseLoader, BaseModel):
    """Load Google Docs from `Google Drive`."""
    service_account_key: Path = Path.home() / ".credentials" / "keys.json"
    """Path to the service account key file."""
    credentials_path: Path = Path.home() / ".credentials" / "credentials.json"
    """Path to the credentials file."""
    token_path: Path = Path.home() / ".credentials" / "token.json"
    """Path to the token file."""
    folder_id: Optional[str] = None
    """The folder id to load from."""
    document_ids: Optional[List[str]] = None
    """The document ids to load from."""
    file_ids: Optional[List[str]] = None
    """The file ids to load from."""
    recursive: bool = False
    """Whether to load recursively. Only applies when folder_id is given."""
"""Whether to load recursively. Only applies when folder_id is given."""
    file_types: Optional[Sequence[str]] = None
    """The file types to load. Only applies when folder_id is given."""
    load_trashed_files: bool = False
    """Whether to load trashed files. Only applies when folder_id is given."""
    # NOTE(MthwRobinson) - changing the file_loader_cls to type here currently
    # results in pydantic validation errors
    file_loader_cls: Any = None
    """The file loader class to use."""
    file_loader_kwargs: Dict["str", Any] = {}
    """The file loader kwargs to use."""
    @root_validator
    def validate_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""
        if values.get("folder_id") and (
            values.get("document_ids") or values.get("file_ids")
        ):
            raise ValueError(
                "Cannot specify both folder_id and document_ids nor "
                "folder_id and file_ids"
            )
        if (
            not values.get("folder_id")
            and not values.get("document_ids")
            and not values.get("file_ids")
        ):
            raise ValueError("Must specify either folder_id, document_ids, or file_ids")
        file_types = values.get("file_types")
        if file_types:
            if values.get("document_ids") or values.get("file_ids"):
                raise ValueError(
                    "file_types can only be given when folder_id is given,"
                    " (not when document_ids or file_ids are given)."
                )
            type_mapping = {
                "document": "application/vnd.google-apps.document",
type_mapping = {
                "document": "application/vnd.google-apps.document",
                "sheet": "application/vnd.google-apps.spreadsheet",
                "pdf": "application/pdf",
            }
            allowed_types = list(type_mapping.keys()) + list(type_mapping.values())
            short_names = ", ".join([f"'{x}'" for x in type_mapping.keys()])
            full_names = ", ".join([f"'{x}'" for x in type_mapping.values()])
            for file_type in file_types:
                if file_type not in allowed_types:
                    raise ValueError(
                        f"Given file type {file_type} is not supported. "
                        f"Supported values are: {short_names}; and "
                        f"their full-form names: {full_names}"
                    )
            # replace short-form file types by full-form file types
            def full_form(x: str) -> str:
                return type_mapping[x] if x in type_mapping else x
            values["file_types"] = [full_form(file_type) for file_type in file_types]
        return values
    @validator("credentials_path")
    def validate_credentials_path(cls, v: Any, **kwargs: Any) -> Any:
        """Validate that credentials_path exists."""
        if not v.exists():
            raise ValueError(f"credentials_path {v} does not exist")
        return v
    def _load_credentials(self) -> Any:
        """Load credentials."""
        # Adapted from https://developers.google.com/drive/api/v3/quickstart/python
        try:
            from google.auth import default
            from google.auth.transport.requests import Request
            from google.oauth2 import service_account
            from google.oauth2.credentials import Credentials
            from google_auth_oauthlib.flow import InstalledAppFlow
        except ImportError:
from google_auth_oauthlib.flow import InstalledAppFlow
        except ImportError:
            raise ImportError(
                "You must run "
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib` "
                "to use the Google Drive loader."
            )
        creds = None
        if self.service_account_key.exists():
            return service_account.Credentials.from_service_account_file(
                str(self.service_account_key), scopes=SCOPES
            )
        if self.token_path.exists():
            creds = Credentials.from_authorized_user_file(str(self.token_path), SCOPES)
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            elif "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ:
                creds, project = default()
                creds = creds.with_scopes(SCOPES)
                # no need to write to file
                if creds:
                    return creds
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    str(self.credentials_path), SCOPES
                )
                creds = flow.run_local_server(port=0)
            with open(self.token_path, "w") as token:
                token.write(creds.to_json())
        return creds
    def _load_sheet_from_id(self, id: str) -> List[Document]:
        """Load a sheet and all tabs from an ID."""
        from googleapiclient.discovery import build
        creds = self._load_credentials()
        sheets_service = build("sheets", "v4", credentials=creds)
        spreadsheet = sheets_service.spreadsheets().get(spreadsheetId=id).execute()
        sheets = spreadsheet.get("sheets", [])
        documents = []
sheets = spreadsheet.get("sheets", [])
        documents = []
        for sheet in sheets:
            sheet_name = sheet["properties"]["title"]
            result = (
                sheets_service.spreadsheets()
                .values()
                .get(spreadsheetId=id, range=sheet_name)
                .execute()
            )
            values = result.get("values", [])
            if not values:
                continue  # empty sheet
            header = values[0]
            for i, row in enumerate(values[1:], start=1):
                metadata = {
                    "source": (
                        f"https://docs.google.com/spreadsheets/d/{id}/"
                        f"edit?gid={sheet['properties']['sheetId']}"
                    ),
                    "title": f"{spreadsheet['properties']['title']} - {sheet_name}",
                    "row": i,
                }
                content = []
                for j, v in enumerate(row):
                    title = header[j].strip() if len(header) > j else ""
                    content.append(f"{title}: {v.strip()}")
                page_content = "\n".join(content)
                documents.append(Document(page_content=page_content, metadata=metadata))
        return documents
    def _load_document_from_id(self, id: str) -> Document:
        """Load a document from an ID."""
        from io import BytesIO
        from googleapiclient.discovery import build
        from googleapiclient.errors import HttpError
        from googleapiclient.http import MediaIoBaseDownload
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        file = (
            service.files()
            .get(fileId=id, supportsAllDrives=True, fields="modifiedTime,name")
            .execute()