viruPVS/langchain-docs huggingface dataset:
Remember to escape special characters if you do not want them to be
                interpreted as regular expression syntax. For example, `.` appears
                frequently in URLs and should be escaped if you want to match a literal
                `.` rather than any character.
                restrict_to_same_domain takes precedence over filter_urls when
                restrict_to_same_domain is True and the sitemap is not a local file.
            parsing_function: Function to parse bs4.Soup output
            blocksize: number of sitemap locations per block
            blocknum: the number of the block that should be loaded - zero indexed.
                Default: 0
            meta_function: Function to parse bs4.Soup output for metadata
                remember when setting this method to also copy metadata["loc"]
                to metadata["source"] if you are using this field
            is_local: whether the sitemap is a local file. Default: False
            continue_on_failure: whether to continue loading the sitemap if an error
                occurs loading a url, emitting a warning instead of raising an
                exception. Setting this to True makes the loader more robust, but also
                may result in missing data. Default: False
            restrict_to_same_domain: whether to restrict loading to URLs to the same
                domain as the sitemap. Attention: This is only applied if the sitemap
                is not a local file!
        """
        if blocksize is not None and blocksize < 1:
            raise ValueError("Sitemap blocksize should be at least 1")
        if blocknum < 0:
            raise ValueError("Sitemap blocknum can not be lower then 0")
        try:
            import lxml  # noqa:F401
        except ImportError:
            raise ImportError(
                "lxml package not found, please install it with `pip install lxml`"
            )
)
        super().__init__(web_paths=[web_path], **kwargs)
        # Define a list of URL patterns (interpreted as regular expressions) that
        # will be allowed to be loaded.
        # restrict_to_same_domain takes precedence over filter_urls when
        # restrict_to_same_domain is True and the sitemap is not a local file.
        self.allow_url_patterns = filter_urls
        self.restrict_to_same_domain = restrict_to_same_domain
        self.parsing_function = parsing_function or _default_parsing_function
        self.meta_function = meta_function or _default_meta_function
        self.blocksize = blocksize
        self.blocknum = blocknum
        self.is_local = is_local
        self.continue_on_failure = continue_on_failure
[docs]    def parse_sitemap(self, soup: Any) -> List[dict]:
        """Parse sitemap xml and load into a list of dicts.
        Args:
            soup: BeautifulSoup object.
        Returns:
            List of dicts.
        """
        els = []
        for url in soup.find_all("url"):
            loc = url.find("loc")
            if not loc:
                continue
            # Strip leading and trailing whitespace and newlines
            loc_text = loc.text.strip()
            if self.restrict_to_same_domain and not self.is_local:
                if _extract_scheme_and_domain(loc_text) != _extract_scheme_and_domain(
                    self.web_path
                ):
                    continue
            if self.allow_url_patterns and not any(
                re.match(regexp_pattern, loc_text)
                for regexp_pattern in self.allow_url_patterns
            ):
                continue
            els.append(
                {
                    tag: prop.text
                    for tag in ["loc", "lastmod", "changefreq", "priority"]
                    if (prop := url.find(tag))
if (prop := url.find(tag))
                }
            )
        for sitemap in soup.find_all("sitemap"):
            loc = sitemap.find("loc")
            if not loc:
                continue
            soup_child = self.scrape_all([loc.text], "xml")[0]
            els.extend(self.parse_sitemap(soup_child))
        return els
[docs]    def load(self) -> List[Document]:
        """Load sitemap."""
        if self.is_local:
            try:
                import bs4
            except ImportError:
                raise ImportError(
                    "beautifulsoup4 package not found, please install it"
                    " with `pip install beautifulsoup4`"
                )
            fp = open(self.web_path)
            soup = bs4.BeautifulSoup(fp, "xml")
        else:
            soup = self._scrape(self.web_path, parser="xml")
        els = self.parse_sitemap(soup)
        if self.blocksize is not None:
            elblocks = list(_batch_block(els, self.blocksize))
            blockcount = len(elblocks)
            if blockcount - 1 < self.blocknum:
                raise ValueError(
                    "Selected sitemap does not contain enough blocks for given blocknum"
                )
            else:
                els = elblocks[self.blocknum]
        results = self.scrape_all([el["loc"].strip() for el in els if "loc" in el])
        return [
            Document(
                page_content=self.parsing_function(results[i]),
                metadata=self.meta_function(els[i], results[i]),
            )
            for i in range(len(results))
        ]
Source code for langchain_community.document_loaders.trello
from __future__ import annotations
from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple
from langchain_core.documents import Document
from langchain_core.utils import get_from_env
from langchain_community.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    from trello import Board, Card, TrelloClient
[docs]class TrelloLoader(BaseLoader):
    """Load cards from a `Trello` board."""
[docs]    def __init__(
        self,
        client: TrelloClient,
        board_name: str,
        *,
        include_card_name: bool = True,
        include_comments: bool = True,
        include_checklist: bool = True,
        card_filter: Literal["closed", "open", "all"] = "all",
        extra_metadata: Tuple[str, ...] = ("due_date", "labels", "list", "closed"),
    ):
        """Initialize Trello loader.
        Args:
            client: Trello API client.
            board_name: The name of the Trello board.
            include_card_name: Whether to include the name of the card in the document.
            include_comments: Whether to include the comments on the card in the
                document.
            include_checklist: Whether to include the checklist on the card in the
                document.
            card_filter: Filter on card status. Valid values are "closed", "open",
                "all".
            extra_metadata: List of additional metadata fields to include as document
                metadata.Valid values are "due_date", "labels", "list", "closed".
        """
        self.client = client
        self.board_name = board_name
        self.include_card_name = include_card_name
        self.include_comments = include_comments
self.include_card_name = include_card_name
        self.include_comments = include_comments
        self.include_checklist = include_checklist
        self.extra_metadata = extra_metadata
        self.card_filter = card_filter
[docs]    @classmethod
    def from_credentials(
        cls,
        board_name: str,
        *,
        api_key: Optional[str] = None,
        token: Optional[str] = None,
        **kwargs: Any,
    ) -> TrelloLoader:
        """Convenience constructor that builds TrelloClient init param for you.
        Args:
            board_name: The name of the Trello board.
            api_key: Trello API key. Can also be specified as environment variable
                TRELLO_API_KEY.
            token: Trello token. Can also be specified as environment variable
                TRELLO_TOKEN.
            include_card_name: Whether to include the name of the card in the document.
            include_comments: Whether to include the comments on the card in the
                document.
            include_checklist: Whether to include the checklist on the card in the
                document.
            card_filter: Filter on card status. Valid values are "closed", "open",
                "all".
            extra_metadata: List of additional metadata fields to include as document
                metadata.Valid values are "due_date", "labels", "list", "closed".
        """
        try:
            from trello import TrelloClient  # type: ignore
        except ImportError as ex:
            raise ImportError(
                "Could not import trello python package. "
                "Please install it with `pip install py-trello`."
            ) from ex
        api_key = api_key or get_from_env("api_key", "TRELLO_API_KEY")
token = token or get_from_env("token", "TRELLO_TOKEN")
        client = TrelloClient(api_key=api_key, token=token)
        return cls(client, board_name, **kwargs)
[docs]    def load(self) -> List[Document]:
        """Loads all cards from the specified Trello board.
        You can filter the cards, metadata and text included by using the optional
            parameters.
         Returns:
            A list of documents, one for each card in the board.
        """
        try:
            from bs4 import BeautifulSoup  # noqa: F401
        except ImportError as ex:
            raise ImportError(
                "`beautifulsoup4` package not found, please run"
                " `pip install beautifulsoup4`"
            ) from ex
        board = self._get_board()
        # Create a dictionary with the list IDs as keys and the list names as values
        list_dict = {list_item.id: list_item.name for list_item in board.list_lists()}
        # Get Cards on the board
        cards = board.get_cards(card_filter=self.card_filter)
        return [self._card_to_doc(card, list_dict) for card in cards]
    def _get_board(self) -> Board:
        # Find the first board with a matching name
        board = next(
            (b for b in self.client.list_boards() if b.name == self.board_name), None
        )
        if not board:
            raise ValueError(f"Board `{self.board_name}` not found.")
        return board
    def _card_to_doc(self, card: Card, list_dict: dict) -> Document:
        from bs4 import BeautifulSoup  # type: ignore
        text_content = ""
        if self.include_card_name:
            text_content = card.name + "\n"
if self.include_card_name:
            text_content = card.name + "\n"
        if card.description.strip():
            text_content += BeautifulSoup(card.description, "lxml").get_text()
        if self.include_checklist:
            # Get all the checklist items on the card
            for checklist in card.checklists:
                if checklist.items:
                    items = [
                        f"{item['name']}:{item['state']}" for item in checklist.items
                    ]
                    text_content += f"\n{checklist.name}\n" + "\n".join(items)
        if self.include_comments:
            # Get all the comments on the card
            comments = [
                BeautifulSoup(comment["data"]["text"], "lxml").get_text()
                for comment in card.comments
            ]
            text_content += "Comments:" + "\n".join(comments)
        # Default metadata fields
        metadata = {
            "title": card.name,
            "id": card.id,
            "url": card.url,
        }
        # Extra metadata fields. Card object is not subscriptable.
        if "labels" in self.extra_metadata:
            metadata["labels"] = [label.name for label in card.labels]
        if "list" in self.extra_metadata:
            if card.list_id in list_dict:
                metadata["list"] = list_dict[card.list_id]
        if "closed" in self.extra_metadata:
            metadata["closed"] = card.closed
        if "due_date" in self.extra_metadata:
            metadata["due_date"] = card.due_date
        return Document(page_content=text_content, metadata=metadata)
Source code for langchain_community.document_loaders.open_city_data
from typing import Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class OpenCityDataLoader(BaseLoader):
    """Load from `Open City`."""
[docs]    def __init__(self, city_id: str, dataset_id: str, limit: int):
        """Initialize with dataset_id.
        Example: https://dev.socrata.com/foundry/data.sfgov.org/vw6y-z8j6
        e.g., city_id = data.sfgov.org
        e.g., dataset_id = vw6y-z8j6
        Args:
            city_id: The Open City city identifier.
            dataset_id: The Open City dataset identifier.
            limit: The maximum number of documents to load.
        """
        self.city_id = city_id
        self.dataset_id = dataset_id
        self.limit = limit
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load records."""
        from sodapy import Socrata
        client = Socrata(self.city_id, None)
        results = client.get(self.dataset_id, limit=self.limit)
        for record in results:
            yield Document(
                page_content=str(record),
                metadata={
                    "source": self.city_id + "_" + self.dataset_id,
                },
            )
[docs]    def load(self) -> List[Document]:
        """Load records."""
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.org_mode
from typing import Any, List
from langchain_community.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredOrgModeLoader(UnstructuredFileLoader):
    """Load `Org-Mode` files using `Unstructured`.
    You can run the loader in one of two modes: "single" and "elements".
    If you use "single" mode, the document will be returned as a single
    langchain Document object. If you use "elements" mode, the unstructured
    library will split the document into elements such as Title and NarrativeText.
    You can pass in additional unstructured kwargs after mode to apply
    different unstructured settings.
    Examples
    --------
    from langchain_community.document_loaders import UnstructuredOrgModeLoader
    loader = UnstructuredOrgModeLoader(
        "example.org", mode="elements", strategy="fast",
    )
    docs = loader.load()
    References
    ----------
    https://unstructured-io.github.io/unstructured/bricks.html#partition-org
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """
        Args:
            file_path: The path to the file to load.
            mode: The mode to load the file from. Default is "single".
            **unstructured_kwargs: Any additional keyword arguments to pass
                to the unstructured.
        """
        validate_unstructured_version(min_unstructured_version="0.7.9")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
Source code for langchain_community.document_loaders.mastodon
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Sequence
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import mastodon
def _dependable_mastodon_import() -> mastodon:
    try:
        import mastodon
    except ImportError:
        raise ImportError(
            "Mastodon.py package not found, "
            "please install it with `pip install Mastodon.py`"
        )
    return mastodon
[docs]class MastodonTootsLoader(BaseLoader):
    """Load the `Mastodon` 'toots'."""
[docs]    def __init__(
        self,
        mastodon_accounts: Sequence[str],
        number_toots: Optional[int] = 100,
        exclude_replies: bool = False,
        access_token: Optional[str] = None,
        api_base_url: str = "https://mastodon.social",
    ):
        """Instantiate Mastodon toots loader.
        Args:
            mastodon_accounts: The list of Mastodon accounts to query.
            number_toots: How many toots to pull for each account. Defaults to 100.
            exclude_replies: Whether to exclude reply toots from the load.
                Defaults to False.
            access_token: An access token if toots are loaded as a Mastodon app. Can
                also be specified via the environment variables "MASTODON_ACCESS_TOKEN".
            api_base_url: A Mastodon API base URL to talk to, if not using the default.
                Defaults to "https://mastodon.social".
        """
Defaults to "https://mastodon.social".
        """
        mastodon = _dependable_mastodon_import()
        access_token = access_token or os.environ.get("MASTODON_ACCESS_TOKEN")
        self.api = mastodon.Mastodon(
            access_token=access_token, api_base_url=api_base_url
        )
        self.mastodon_accounts = mastodon_accounts
        self.number_toots = number_toots
        self.exclude_replies = exclude_replies
[docs]    def load(self) -> List[Document]:
        """Load toots into documents."""
        results: List[Document] = []
        for account in self.mastodon_accounts:
            user = self.api.account_lookup(account)
            toots = self.api.account_statuses(
                user.id,
                only_media=False,
                pinned=False,
                exclude_replies=self.exclude_replies,
                exclude_reblogs=True,
                limit=self.number_toots,
            )
            docs = self._format_toots(toots, user)
            results.extend(docs)
        return results
    def _format_toots(
        self, toots: List[Dict[str, Any]], user_info: dict
    ) -> Iterable[Document]:
        """Format toots into documents.
        Adding user info, and selected toot fields into the metadata.
        """
        for toot in toots:
            metadata = {
                "created_at": toot["created_at"],
                "user_info": user_info,
                "is_reply": toot["in_reply_to_id"] is not None,
            }
            yield Document(
                page_content=toot["content"],
                metadata=metadata,
            )
Source code for langchain_community.document_loaders.azlyrics
from typing import List
from langchain_core.documents import Document
from langchain_community.document_loaders.web_base import WebBaseLoader
[docs]class AZLyricsLoader(WebBaseLoader):
    """Load `AZLyrics` webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpages into Documents."""
        soup = self.scrape()
        title = soup.title.text
        lyrics = soup.find_all("div", {"class": ""})[2].text
        text = title + lyrics
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
Source code for langchain_community.document_loaders.web_base
"""Web base loader class."""
import asyncio
import logging
import warnings
from typing import Any, Dict, Iterator, List, Optional, Sequence, Union
import aiohttp
import requests
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
default_header_template = {
    "User-Agent": "",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
    ";q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Referer": "https://www.google.com/",
    "DNT": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
}
def _build_metadata(soup: Any, url: str) -> dict:
    """Build metadata from BeautifulSoup output."""
    metadata = {"source": url}
    if title := soup.find("title"):
        metadata["title"] = title.get_text()
    if description := soup.find("meta", attrs={"name": "description"}):
        metadata["description"] = description.get("content", "No description found.")
    if html := soup.find("html"):
        metadata["language"] = html.get("lang", "No language found.")
    return metadata
[docs]class WebBaseLoader(BaseLoader):
    """Load HTML pages using `urllib` and parse them with `BeautifulSoup'."""
[docs]    def __init__(
        self,
        web_path: Union[str, Sequence[str]] = "",
        header_template: Optional[dict] = None,
        verify_ssl: bool = True,
        proxies: Optional[dict] = None,
verify_ssl: bool = True,
        proxies: Optional[dict] = None,
        continue_on_failure: bool = False,
        autoset_encoding: bool = True,
        encoding: Optional[str] = None,
        web_paths: Sequence[str] = (),
        requests_per_second: int = 2,
        default_parser: str = "html.parser",
        requests_kwargs: Optional[Dict[str, Any]] = None,
        raise_for_status: bool = False,
        bs_get_text_kwargs: Optional[Dict[str, Any]] = None,
        bs_kwargs: Optional[Dict[str, Any]] = None,
        session: Any = None,
    ) -> None:
        """Initialize loader.
        Args:
            web_paths: Web paths to load from.
            requests_per_second: Max number of concurrent requests to make.
            default_parser: Default parser to use for BeautifulSoup.
            requests_kwargs: kwargs for requests
            raise_for_status: Raise an exception if http status code denotes an error.
            bs_get_text_kwargs: kwargs for beatifulsoup4 get_text
            bs_kwargs: kwargs for beatifulsoup4 web page parsing
        """
        # web_path kept for backwards-compatibility.
        if web_path and web_paths:
            raise ValueError(
                "Received web_path and web_paths. Only one can be specified. "
                "web_path is deprecated, web_paths should be used."
            )
        if web_paths:
            self.web_paths = list(web_paths)
        elif isinstance(web_path, str):
            self.web_paths = [web_path]
        elif isinstance(web_path, Sequence):
            self.web_paths = list(web_path)
        else:
            raise TypeError(
                f"web_path must be str or Sequence[str] got ({type(web_path)}) or"
f" web_paths must be Sequence[str] got ({type(web_paths)})"
            )
        self.requests_per_second = requests_per_second
        self.default_parser = default_parser
        self.requests_kwargs = requests_kwargs or {}
        self.raise_for_status = raise_for_status
        self.bs_get_text_kwargs = bs_get_text_kwargs or {}
        self.bs_kwargs = bs_kwargs or {}
        if session:
            self.session = session
        else:
            session = requests.Session()
            header_template = header_template or default_header_template.copy()
            if not header_template.get("User-Agent"):
                try:
                    from fake_useragent import UserAgent
                    header_template["User-Agent"] = UserAgent().random
                except ImportError:
                    logger.info(
                        "fake_useragent not found, using default user agent."
                        "To get a realistic header for requests, "
                        "`pip install fake_useragent`."
                    )
            session.headers = dict(header_template)
            session.verify = verify_ssl
            if proxies:
                session.proxies.update(proxies)
            self.session = session
        self.continue_on_failure = continue_on_failure
        self.autoset_encoding = autoset_encoding
        self.encoding = encoding
    @property
    def web_path(self) -> str:
        if len(self.web_paths) > 1:
            raise ValueError("Multiple webpaths found.")
        return self.web_paths[0]
    async def _fetch(
        self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5
    ) -> str:
        async with aiohttp.ClientSession() as session:
            for i in range(retries):
                try:
                    async with session.get(
                        url,
                        headers=self.session.headers,