viruPVS/langchain-docs huggingface dataset:
def _parse_dgml(
        self,
        content: bytes,
        document_name: Optional[str] = None,
        additional_doc_metadata: Optional[Mapping] = None,
    ) -> List[Document]:
        """Parse a single DGML document into a list of Documents."""
        try:
            from lxml import etree
        except ImportError:
            raise ImportError(
                "Could not import lxml python package. "
                "Please install it with `pip install lxml`."
            )
        try:
            from dgml_utils.models import Chunk
            from dgml_utils.segmentation import get_chunks
        except ImportError:
            raise ImportError(
                "Could not import from dgml-utils python package. "
                "Please install it with `pip install dgml-utils`."
            )
        def _build_framework_chunk(dg_chunk: Chunk) -> Document:
            # Stable IDs for chunks with the same text.
            _hashed_id = hashlib.md5(dg_chunk.text.encode()).hexdigest()
            metadata = {
                XPATH_KEY: dg_chunk.xpath,
                ID_KEY: _hashed_id,
                DOCUMENT_NAME_KEY: document_name,
                DOCUMENT_SOURCE_KEY: document_name,
                STRUCTURE_KEY: dg_chunk.structure,
                TAG_KEY: dg_chunk.tag,
            }
            text = dg_chunk.text
            if additional_doc_metadata:
                if self.include_project_metadata_in_doc_metadata:
                    metadata.update(additional_doc_metadata)
            return Document(
                page_content=text[: self.max_text_length],
                metadata=metadata,
            )
        # Parse the tree and return chunks
        tree = etree.parse(io.BytesIO(content))
        root = tree.getroot()
        dg_chunks = get_chunks(
            root,
            min_text_length=self.min_text_length,
root,
            min_text_length=self.min_text_length,
            max_text_length=self.max_text_length,
            whitespace_normalize_text=self.whitespace_normalize_text,
            sub_chunk_tables=self.sub_chunk_tables,
            include_xml_tags=self.include_xml_tags,
            parent_hierarchy_levels=self.parent_hierarchy_levels,
        )
        framework_chunks: Dict[str, Document] = {}
        for dg_chunk in dg_chunks:
            framework_chunk = _build_framework_chunk(dg_chunk)
            chunk_id = framework_chunk.metadata.get(ID_KEY)
            if chunk_id:
                framework_chunks[chunk_id] = framework_chunk
                if dg_chunk.parent:
                    framework_parent_chunk = _build_framework_chunk(dg_chunk.parent)
                    parent_id = framework_parent_chunk.metadata.get(ID_KEY)
                    if parent_id and framework_parent_chunk.page_content:
                        framework_chunk.metadata[self.parent_id_key] = parent_id
                        framework_chunks[parent_id] = framework_parent_chunk
        return list(framework_chunks.values())
    def _document_details_for_docset_id(self, docset_id: str) -> List[Dict]:
        """Gets all document details for the given docset ID"""
        url = f"{self.api}/docsets/{docset_id}/documents"
        all_documents = []
        while url:
            response = requests.get(
                url,
                headers={"Authorization": f"Bearer {self.access_token}"},
            )
            if response.ok:
                data = response.json()
                all_documents.extend(data["documents"])
                url = data.get("next", None)
            else:
                raise Exception(
                    f"Failed to download {url} (status: {response.status_code})"
                )
        return all_documents
    def _project_details_for_docset_id(self, docset_id: str) -> List[Dict]:
"""Gets all project details for the given docset ID"""
        url = f"{self.api}/projects?docset.id={docset_id}"
        all_projects = []
        while url:
            response = requests.request(
                "GET",
                url,
                headers={"Authorization": f"Bearer {self.access_token}"},
                data={},
            )
            if response.ok:
                data = response.json()
                all_projects.extend(data["projects"])
                url = data.get("next", None)
            else:
                raise Exception(
                    f"Failed to download {url} (status: {response.status_code})"
                )
        return all_projects
    def _metadata_for_project(self, project: Dict) -> Dict:
        """Gets project metadata for all files"""
        project_id = project.get(ID_KEY)
        url = f"{self.api}/projects/{project_id}/artifacts/latest"
        all_artifacts = []
        per_file_metadata: Dict = {}
        while url:
            response = requests.request(
                "GET",
                url,
                headers={"Authorization": f"Bearer {self.access_token}"},
                data={},
            )
            if response.ok:
                data = response.json()
                all_artifacts.extend(data["artifacts"])
                url = data.get("next", None)
            elif response.status_code == 404:
                # Not found is ok, just means no published projects
                return per_file_metadata
            else:
                raise Exception(
                    f"Failed to download {url} (status: {response.status_code})"
                )
        for artifact in all_artifacts:
            artifact_name = artifact.get("name")
            artifact_url = artifact.get("url")
            artifact_doc = artifact.get("document")
artifact_url = artifact.get("url")
            artifact_doc = artifact.get("document")
            if artifact_name == "report-values.xml" and artifact_url and artifact_doc:
                doc_id = artifact_doc[ID_KEY]
                metadata: Dict = {}
                # The evaluated XML for each document is named after the project
                response = requests.request(
                    "GET",
                    f"{artifact_url}/content",
                    headers={"Authorization": f"Bearer {self.access_token}"},
                    data={},
                )
                if response.ok:
                    try:
                        from lxml import etree
                    except ImportError:
                        raise ImportError(
                            "Could not import lxml python package. "
                            "Please install it with `pip install lxml`."
                        )
                    artifact_tree = etree.parse(io.BytesIO(response.content))
                    artifact_root = artifact_tree.getroot()
                    ns = artifact_root.nsmap
                    entries = artifact_root.xpath("//pr:Entry", namespaces=ns)
                    for entry in entries:
                        heading = entry.xpath("./pr:Heading", namespaces=ns)[0].text
                        value = " ".join(
                            entry.xpath("./pr:Value", namespaces=ns)[0].itertext()
                        ).strip()
                        metadata[heading] = value[: self.max_metadata_length]
                    per_file_metadata[doc_id] = metadata
                else:
                    raise Exception(
                        f"Failed to download {artifact_url}/content "
                        + "(status: {response.status_code})"
                    )
        return per_file_metadata
    def _load_chunks_for_document(
        self,
        document_id: str,
        docset_id: str,
        document_name: Optional[str] = None,
        additional_metadata: Optional[Mapping] = None,
    ) -> List[Document]:
additional_metadata: Optional[Mapping] = None,
    ) -> List[Document]:
        """Load chunks for a document."""
        url = f"{self.api}/docsets/{docset_id}/documents/{document_id}/dgml"
        response = requests.request(
            "GET",
            url,
            headers={"Authorization": f"Bearer {self.access_token}"},
            data={},
        )
        if response.ok:
            return self._parse_dgml(
                content=response.content,
                document_name=document_name,
                additional_doc_metadata=additional_metadata,
            )
        else:
            raise Exception(
                f"Failed to download {url} (status: {response.status_code})"
            )
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        chunks: List[Document] = []
        if self.access_token and self.docset_id:
            # Remote mode
            _document_details = self._document_details_for_docset_id(self.docset_id)
            if self.document_ids:
                _document_details = [
                    d for d in _document_details if d[ID_KEY] in self.document_ids
                ]
            _project_details = self._project_details_for_docset_id(self.docset_id)
            combined_project_metadata: Dict[str, Dict] = {}
            if _project_details and self.include_project_metadata_in_doc_metadata:
                # If there are any projects for this docset and the caller requested
                # project metadata, load it.
                for project in _project_details:
                    metadata = self._metadata_for_project(project)
                    for file_id in metadata:
                        if file_id not in combined_project_metadata:
                            combined_project_metadata[file_id] = metadata[file_id]
                        else:
combined_project_metadata[file_id] = metadata[file_id]
                        else:
                            combined_project_metadata[file_id].update(metadata[file_id])
            for doc in _document_details:
                doc_id = doc[ID_KEY]
                doc_name = doc.get(DOCUMENT_NAME_KEY)
                doc_metadata = combined_project_metadata.get(doc_id)
                chunks += self._load_chunks_for_document(
                    document_id=doc_id,
                    docset_id=self.docset_id,
                    document_name=doc_name,
                    additional_metadata=doc_metadata,
                )
        elif self.file_paths:
            # Local mode (for integration testing, or pre-downloaded XML)
            for path in self.file_paths:
                path = Path(path)
                with open(path, "rb") as file:
                    chunks += self._parse_dgml(
                        content=file.read(),
                        document_name=path.name,
                    )
        return chunks
Source code for langchain_community.document_loaders.reddit
from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import praw
def _dependable_praw_import() -> praw:
    try:
        import praw
    except ImportError:
        raise ImportError(
            "praw package not found, please install it with `pip install praw`"
        )
    return praw
[docs]class RedditPostsLoader(BaseLoader):
    """Load `Reddit` posts.
    Read posts on a subreddit.
    First, you need to go to
    https://www.reddit.com/prefs/apps/
    and create your application
    """
[docs]    def __init__(
        self,
        client_id: str,
        client_secret: str,
        user_agent: str,
        search_queries: Sequence[str],
        mode: str,
        categories: Sequence[str] = ["new"],
        number_posts: Optional[int] = 10,
    ):
        """
        Initialize with client_id, client_secret, user_agent, search_queries, mode,
            categories, number_posts.
        Example: https://www.reddit.com/r/learnpython/
        Args:
            client_id: Reddit client id.
            client_secret: Reddit client secret.
            user_agent: Reddit user agent.
            search_queries: The search queries.
            mode: The mode.
            categories: The categories. Default: ["new"]
            number_posts: The number of posts. Default: 10
        """
        self.client_id = client_id
        self.client_secret = client_secret
        self.user_agent = user_agent
        self.search_queries = search_queries
self.user_agent = user_agent
        self.search_queries = search_queries
        self.mode = mode
        self.categories = categories
        self.number_posts = number_posts
[docs]    def load(self) -> List[Document]:
        """Load reddits."""
        praw = _dependable_praw_import()
        reddit = praw.Reddit(
            client_id=self.client_id,
            client_secret=self.client_secret,
            user_agent=self.user_agent,
        )
        results: List[Document] = []
        if self.mode == "subreddit":
            for search_query in self.search_queries:
                for category in self.categories:
                    docs = self._subreddit_posts_loader(
                        search_query=search_query, category=category, reddit=reddit
                    )
                    results.extend(docs)
        elif self.mode == "username":
            for search_query in self.search_queries:
                for category in self.categories:
                    docs = self._user_posts_loader(
                        search_query=search_query, category=category, reddit=reddit
                    )
                    results.extend(docs)
        else:
            raise ValueError(
                "mode not correct, please enter 'username' or 'subreddit' as mode"
            )
        return results
    def _subreddit_posts_loader(
        self, search_query: str, category: str, reddit: praw.reddit.Reddit
    ) -> Iterable[Document]:
        subreddit = reddit.subreddit(search_query)
        method = getattr(subreddit, category)
        cat_posts = method(limit=self.number_posts)
        """Format reddit posts into a string."""
        for post in cat_posts:
            metadata = {
                "post_subreddit": post.subreddit_name_prefixed,
                "post_category": category,
                "post_title": post.title,
"post_category": category,
                "post_title": post.title,
                "post_score": post.score,
                "post_id": post.id,
                "post_url": post.url,
                "post_author": post.author,
            }
            yield Document(
                page_content=post.selftext,
                metadata=metadata,
            )
    def _user_posts_loader(
        self, search_query: str, category: str, reddit: praw.reddit.Reddit
    ) -> Iterable[Document]:
        user = reddit.redditor(search_query)
        method = getattr(user.submissions, category)
        cat_posts = method(limit=self.number_posts)
        """Format reddit posts into a string."""
        for post in cat_posts:
            metadata = {
                "post_subreddit": post.subreddit_name_prefixed,
                "post_category": category,
                "post_title": post.title,
                "post_score": post.score,
                "post_id": post.id,
                "post_url": post.url,
                "post_author": post.author,
            }
            yield Document(
                page_content=post.selftext,
                metadata=metadata,
            )
Source code for langchain_community.document_loaders.couchbase
import logging
from typing import Iterator, List, Optional
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class CouchbaseLoader(BaseLoader):
    """Load documents from `Couchbase`.
    Each document represents one row of the result. The `page_content_fields` are
    written into the `page_content`of the document. The `metadata_fields` are written
    into the `metadata` of the document. By default, all columns are written into
    the `page_content` and none into the `metadata`.
    """
[docs]    def __init__(
        self,
        connection_string: str,
        db_username: str,
        db_password: str,
        query: str,
        *,
        page_content_fields: Optional[List[str]] = None,
        metadata_fields: Optional[List[str]] = None,
    ) -> None:
        """Initialize Couchbase document loader.
        Args:
            connection_string (str): The connection string to the Couchbase cluster.
            db_username (str): The username to connect to the Couchbase cluster.
            db_password (str): The password to connect to the Couchbase cluster.
            query (str): The SQL++ query to execute.
            page_content_fields (Optional[List[str]]): The columns to write into the
                `page_content` field of the document. By default, all columns are
                written.
            metadata_fields (Optional[List[str]]): The columns to write into the
                `metadata` field of the document. By default, no columns are written.
        """
        try:
            from couchbase.auth import PasswordAuthenticator
            from couchbase.cluster import Cluster
from couchbase.auth import PasswordAuthenticator
            from couchbase.cluster import Cluster
            from couchbase.options import ClusterOptions
        except ImportError as e:
            raise ImportError(
                "Could not import couchbase package."
                "Please install couchbase SDK with `pip install couchbase`."
            ) from e
        if not connection_string:
            raise ValueError("connection_string must be provided.")
        if not db_username:
            raise ValueError("db_username must be provided.")
        if not db_password:
            raise ValueError("db_password must be provided.")
        auth = PasswordAuthenticator(
            db_username,
            db_password,
        )
        self.cluster: Cluster = Cluster(connection_string, ClusterOptions(auth))
        self.query = query
        self.page_content_fields = page_content_fields
        self.metadata_fields = metadata_fields
[docs]    def load(self) -> List[Document]:
        """Load Couchbase data into Document objects."""
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load Couchbase data into Document objects lazily."""
        from datetime import timedelta
        # Ensure connection to Couchbase cluster
        self.cluster.wait_until_ready(timedelta(seconds=5))
        # Run SQL++ Query
        result = self.cluster.query(self.query)
        for row in result:
            metadata_fields = self.metadata_fields
            page_content_fields = self.page_content_fields
            if not page_content_fields:
                page_content_fields = list(row.keys())
            if not metadata_fields:
                metadata_fields = []
            metadata = {field: row[field] for field in metadata_fields}
            document = "\n".join(
                f"{k}: {v}" for k, v in row.items() if k in page_content_fields
            )
Source code for langchain_community.document_loaders.onedrive
"""Loads data from OneDrive"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Iterator, List, Optional, Sequence, Union
from langchain_core.documents import Document
from langchain_core.pydantic_v1 import Field
from langchain_community.document_loaders.base_o365 import (
    O365BaseLoader,
    _FileType,
)
from langchain_community.document_loaders.parsers.registry import get_parser
if TYPE_CHECKING:
    from O365.drive import Drive, Folder
logger = logging.getLogger(__name__)
[docs]class OneDriveLoader(O365BaseLoader):
    """Load from `Microsoft OneDrive`."""
    drive_id: str = Field(...)
    """ The ID of the OneDrive drive to load data from."""
    folder_path: Optional[str] = None
    """ The path to the folder to load data from."""
    object_ids: Optional[List[str]] = None
    """ The IDs of the objects to load data from."""
    @property
    def _file_types(self) -> Sequence[_FileType]:
        """Return supported file types."""
        return _FileType.DOC, _FileType.DOCX, _FileType.PDF
    @property
    def _scopes(self) -> List[str]:
        """Return required scopes."""
        return ["offline_access", "Files.Read.All"]
    def _get_folder_from_path(self, drive: Drive) -> Union[Folder, Drive]:
        """
        Returns the folder or drive object located at the
        specified path relative to the given drive.
        Args:
            drive (Drive): The root drive from which the folder path is relative.
        Returns:
            Union[Folder, Drive]: The folder or drive object
            located at the specified path.
        Raises:
located at the specified path.
        Raises:
            FileNotFoundError: If the path does not exist.
        """
        subfolder_drive = drive
        if self.folder_path is None:
            return subfolder_drive
        subfolders = [f for f in self.folder_path.split("/") if f != ""]
        if len(subfolders) == 0:
            return subfolder_drive
        items = subfolder_drive.get_items()
        for subfolder in subfolders:
            try:
                subfolder_drive = list(filter(lambda x: subfolder in x.name, items))[0]
                items = subfolder_drive.get_items()
            except (IndexError, AttributeError):
                raise FileNotFoundError("Path {} not exist.".format(self.folder_path))
        return subfolder_drive
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load documents lazily. Use this when working at a large scale."""
        try:
            from O365.drive import Drive
        except ImportError:
            raise ImportError(
                "O365 package not found, please install it with `pip install o365`"
            )
        drive = self._auth().storage().get_drive(self.drive_id)
        if not isinstance(drive, Drive):
            raise ValueError(f"There isn't a Drive with id {self.drive_id}.")
        blob_parser = get_parser("default")
        if self.folder_path:
            folder = self._get_folder_from_path(drive)
            for blob in self._load_from_folder(folder):
                yield from blob_parser.lazy_parse(blob)
        if self.object_ids:
            for blob in self._load_from_object_ids(drive, self.object_ids):
                yield from blob_parser.lazy_parse(blob)
[docs]    def load(self) -> List[Document]:
        """Load all documents."""
Source code for langchain_community.document_loaders.hugging_face_dataset
import json
from typing import Iterator, List, Mapping, Optional, Sequence, Union
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class HuggingFaceDatasetLoader(BaseLoader):
    """Load from `Hugging Face Hub` datasets."""
[docs]    def __init__(
        self,
        path: str,
        page_content_column: str = "text",
        name: Optional[str] = None,
        data_dir: Optional[str] = None,
        data_files: Optional[
            Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
        ] = None,
        cache_dir: Optional[str] = None,
        keep_in_memory: Optional[bool] = None,
        save_infos: bool = False,
        use_auth_token: Optional[Union[bool, str]] = None,
        num_proc: Optional[int] = None,
    ):
        """Initialize the HuggingFaceDatasetLoader.
        Args:
            path: Path or name of the dataset.
            page_content_column: Page content column name. Default is "text".
            name: Name of the dataset configuration.
            data_dir: Data directory of the dataset configuration.
            data_files: Path(s) to source data file(s).
            cache_dir: Directory to read/write data.
            keep_in_memory: Whether to copy the dataset in-memory.
            save_infos: Save the dataset information (checksums/size/splits/...).
              Default is False.
            use_auth_token: Bearer token for remote files on the Dataset Hub.
            num_proc: Number of processes.
        """
        self.path = path
        self.page_content_column = page_content_column