viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.document_loaders.larksuite
import json
import urllib.request
from typing import Any, Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class LarkSuiteDocLoader(BaseLoader):
    """Load from `LarkSuite` (`FeiShu`)."""
[docs]    def __init__(self, domain: str, access_token: str, document_id: str):
        """Initialize with domain, access_token (tenant / user), and document_id.
        Args:
            domain: The domain to load the LarkSuite.
            access_token: The access_token to use.
            document_id: The document_id to load.
        """
        self.domain = domain
        self.access_token = access_token
        self.document_id = document_id
    def _get_larksuite_api_json_data(self, api_url: str) -> Any:
        """Get LarkSuite (FeiShu) API response json data."""
        headers = {"Authorization": f"Bearer {self.access_token}"}
        request = urllib.request.Request(api_url, headers=headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            return json_data
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load LarkSuite (FeiShu) document."""
        api_url_prefix = f"{self.domain}/open-apis/docx/v1/documents"
        metadata_json = self._get_larksuite_api_json_data(
            f"{api_url_prefix}/{self.document_id}"
        )
        raw_content_json = self._get_larksuite_api_json_data(
            f"{api_url_prefix}/{self.document_id}/raw_content"
        )
f"{api_url_prefix}/{self.document_id}/raw_content"
        )
        text = raw_content_json["data"]["content"]
        metadata = {
            "document_id": self.document_id,
            "revision_id": metadata_json["data"]["document"]["revision_id"],
            "title": metadata_json["data"]["document"]["title"],
        }
        yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load LarkSuite (FeiShu) document."""
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.azure_ai_data
from typing import Iterator, List, Optional
from langchain_community.docstore.document import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.unstructured import UnstructuredFileIOLoader
[docs]class AzureAIDataLoader(BaseLoader):
    """Load from Azure AI Data."""
[docs]    def __init__(self, url: str, glob: Optional[str] = None):
        """Initialize with URL to a data asset or storage location
        ."""
        self.url = url
        """URL to the data asset or storage location."""
        self.glob_pattern = glob
        """Optional glob pattern to select files. Defaults to None."""
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader for Documents."""
        try:
            from azureml.fsspec import AzureMachineLearningFileSystem
        except ImportError as exc:
            raise ImportError(
                "Could not import azureml-fspec package."
                "Please install it with `pip install azureml-fsspec`."
            ) from exc
        fs = AzureMachineLearningFileSystem(self.url)
        if self.glob_pattern:
            remote_paths_list = fs.glob(self.glob_pattern)
        else:
            remote_paths_list = fs.ls()
        for remote_path in remote_paths_list:
            with fs.open(remote_path) as f:
                loader = UnstructuredFileIOLoader(file=f)
                yield from loader.load()
Source code for langchain_community.document_loaders.ifixit
from typing import List, Optional
import requests
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.web_base import WebBaseLoader
IFIXIT_BASE_URL = "https://www.ifixit.com/api/2.0"
[docs]class IFixitLoader(BaseLoader):
    """Load `iFixit` repair guides, device wikis and answers.
    iFixit is the largest, open repair community on the web. The site contains nearly
    100k repair manuals, 200k Questions & Answers on 42k devices, and all the data is
    licensed under CC-BY.
    This loader will allow you to download the text of a repair guide, text of Q&A's
    and wikis from devices on iFixit using their open APIs and web scraping.
    """
[docs]    def __init__(self, web_path: str):
        """Initialize with a web path."""
        if not web_path.startswith("https://www.ifixit.com"):
            raise ValueError("web path must start with 'https://www.ifixit.com'")
        path = web_path.replace("https://www.ifixit.com", "")
        allowed_paths = ["/Device", "/Guide", "/Answers", "/Teardown"]
        """ TODO: Add /Wiki """
        if not any(path.startswith(allowed_path) for allowed_path in allowed_paths):
            raise ValueError(
                "web path must start with /Device, /Guide, /Teardown or /Answers"
            )
        pieces = [x for x in path.split("/") if x]
        """Teardowns are just guides by a different name"""
"""Teardowns are just guides by a different name"""
        self.page_type = pieces[0] if pieces[0] != "Teardown" else "Guide"
        if self.page_type == "Guide" or self.page_type == "Answers":
            self.id = pieces[2]
        else:
            self.id = pieces[1]
        self.web_path = web_path
[docs]    def load(self) -> List[Document]:
        if self.page_type == "Device":
            return self.load_device()
        elif self.page_type == "Guide" or self.page_type == "Teardown":
            return self.load_guide()
        elif self.page_type == "Answers":
            return self.load_questions_and_answers()
        else:
            raise ValueError("Unknown page type: " + self.page_type)
[docs]    @staticmethod
    def load_suggestions(query: str = "", doc_type: str = "all") -> List[Document]:
        """Load suggestions.
        Args:
            query: A query string
            doc_type: The type of document to search for. Can be one of "all",
              "device", "guide", "teardown", "answer", "wiki".
        Returns:
        """
        res = requests.get(
            IFIXIT_BASE_URL + "/suggest/" + query + "?doctypes=" + doc_type
        )
        if res.status_code != 200:
            raise ValueError(
                'Could not load suggestions for "' + query + '"\n' + res.json()
            )
        data = res.json()
        results = data["results"]
        output = []
        for result in results:
            try:
                loader = IFixitLoader(result["url"])
                if loader.page_type == "Device":
if loader.page_type == "Device":
                    output += loader.load_device(include_guides=False)
                else:
                    output += loader.load()
            except ValueError:
                continue
        return output
[docs]    def load_questions_and_answers(
        self, url_override: Optional[str] = None
    ) -> List[Document]:
        """Load a list of questions and answers.
        Args:
            url_override: A URL to override the default URL.
        Returns: List[Document]
        """
        loader = WebBaseLoader(self.web_path if url_override is None else url_override)
        soup = loader.scrape()
        output = []
        title = soup.find("h1", "post-title").text
        output.append("# " + title)
        output.append(soup.select_one(".post-content .post-text").text.strip())
        answersHeader = soup.find("div", "post-answers-header")
        if answersHeader:
            output.append("\n## " + answersHeader.text.strip())
        for answer in soup.select(".js-answers-list .post.post-answer"):
            if answer.has_attr("itemprop") and "acceptedAnswer" in answer["itemprop"]:
                output.append("\n### Accepted Answer")
            elif "post-helpful" in answer["class"]:
                output.append("\n### Most Helpful Answer")
            else:
                output.append("\n### Other Answer")
            output += [
                a.text.strip() for a in answer.select(".post-content .post-text")
            ]
            output.append("\n")
        text = "\n".join(output).strip()
        metadata = {"source": self.web_path, "title": title}
        return [Document(page_content=text, metadata=metadata)]
[docs]    def load_device(
[docs]    def load_device(
        self, url_override: Optional[str] = None, include_guides: bool = True
    ) -> List[Document]:
        """Loads a device
        Args:
            url_override: A URL to override the default URL.
            include_guides: Whether to include guides linked to from the device.
              Defaults to True.
        Returns:
        """
        documents = []
        if url_override is None:
            url = IFIXIT_BASE_URL + "/wikis/CATEGORY/" + self.id
        else:
            url = url_override
        res = requests.get(url)
        data = res.json()
        text = "\n".join(
            [
                data[key]
                for key in ["title", "description", "contents_raw"]
                if key in data
            ]
        ).strip()
        metadata = {"source": self.web_path, "title": data["title"]}
        documents.append(Document(page_content=text, metadata=metadata))
        if include_guides:
            """Load and return documents for each guide linked to from the device"""
            guide_urls = [guide["url"] for guide in data["guides"]]
            for guide_url in guide_urls:
                documents.append(IFixitLoader(guide_url).load()[0])
        return documents
[docs]    def load_guide(self, url_override: Optional[str] = None) -> List[Document]:
        """Load a guide
        Args:
            url_override: A URL to override the default URL.
        Returns: List[Document]
        """
        if url_override is None:
            url = IFIXIT_BASE_URL + "/guides/" + self.id
        else:
            url = url_override
        res = requests.get(url)
else:
            url = url_override
        res = requests.get(url)
        if res.status_code != 200:
            raise ValueError(
                "Could not load guide: " + self.web_path + "\n" + res.json()
            )
        data = res.json()
        doc_parts = ["# " + data["title"], data["introduction_raw"]]
        doc_parts.append("\n\n###Tools Required:")
        if len(data["tools"]) == 0:
            doc_parts.append("\n - None")
        else:
            for tool in data["tools"]:
                doc_parts.append("\n - " + tool["text"])
        doc_parts.append("\n\n###Parts Required:")
        if len(data["parts"]) == 0:
            doc_parts.append("\n - None")
        else:
            for part in data["parts"]:
                doc_parts.append("\n - " + part["text"])
        for row in data["steps"]:
            doc_parts.append(
                "\n\n## "
                + (
                    row["title"]
                    if row["title"] != ""
                    else "Step {}".format(row["orderby"])
                )
            )
            for line in row["lines"]:
                doc_parts.append(line["text_raw"])
        doc_parts.append(data["conclusion_raw"])
        text = "\n".join(doc_parts)
        metadata = {"source": self.web_path, "title": data["title"]}
        return [Document(page_content=text, metadata=metadata)]
Source code for langchain_community.document_loaders.csv_loader
import csv
from io import TextIOWrapper
from typing import Any, Dict, List, Optional, Sequence
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.helpers import detect_file_encodings
from langchain_community.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class CSVLoader(BaseLoader):
    """Load a `CSV` file into a list of Documents.
    Each document represents one row of the CSV file. Every row is converted into a
    key/value pair and outputted to a new line in the document's page_content.
    The source for each document loaded from csv is set to the value of the
    `file_path` argument for all documents by default.
    You can override this by setting the `source_column` argument to the
    name of a column in the CSV file.
    The source of each document will then be set to the value of the column
    with the name specified in `source_column`.
    Output Example:
        .. code-block:: txt
            column1: value1
            column2: value2
            column3: value3
    """
[docs]    def __init__(
        self,
        file_path: str,
        source_column: Optional[str] = None,
        metadata_columns: Sequence[str] = (),
        csv_args: Optional[Dict] = None,
        encoding: Optional[str] = None,
        autodetect_encoding: bool = False,
    ):
        """
        Args:
            file_path: The path to the CSV file.
            source_column: The name of the column in the CSV file to use as the source.
source_column: The name of the column in the CSV file to use as the source.
              Optional. Defaults to None.
            metadata_columns: A sequence of column names to use as metadata. Optional.
            csv_args: A dictionary of arguments to pass to the csv.DictReader.
              Optional. Defaults to None.
            encoding: The encoding of the CSV file. Optional. Defaults to None.
            autodetect_encoding: Whether to try to autodetect the file encoding.
        """
        self.file_path = file_path
        self.source_column = source_column
        self.metadata_columns = metadata_columns
        self.encoding = encoding
        self.csv_args = csv_args or {}
        self.autodetect_encoding = autodetect_encoding
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        docs = []
        try:
            with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
                docs = self.__read_file(csvfile)
        except UnicodeDecodeError as e:
            if self.autodetect_encoding:
                detected_encodings = detect_file_encodings(self.file_path)
                for encoding in detected_encodings:
                    try:
                        with open(
                            self.file_path, newline="", encoding=encoding.encoding
                        ) as csvfile:
                            docs = self.__read_file(csvfile)
                            break
                    except UnicodeDecodeError:
                        continue
            else:
                raise RuntimeError(f"Error loading {self.file_path}") from e
        except Exception as e:
            raise RuntimeError(f"Error loading {self.file_path}") from e
        return docs
    def __read_file(self, csvfile: TextIOWrapper) -> List[Document]:
        docs = []
docs = []
        csv_reader = csv.DictReader(csvfile, **self.csv_args)  # type: ignore
        for i, row in enumerate(csv_reader):
            try:
                source = (
                    row[self.source_column]
                    if self.source_column is not None
                    else self.file_path
                )
            except KeyError:
                raise ValueError(
                    f"Source column '{self.source_column}' not found in CSV file."
                )
            content = "\n".join(
                f"{k.strip()}: {v.strip() if v is not None else v}"
                for k, v in row.items()
                if k not in self.metadata_columns
            )
            metadata = {"source": source, "row": i}
            for col in self.metadata_columns:
                try:
                    metadata[col] = row[col]
                except KeyError:
                    raise ValueError(f"Metadata column '{col}' not found in CSV file.")
            doc = Document(page_content=content, metadata=metadata)
            docs.append(doc)
        return docs
[docs]class UnstructuredCSVLoader(UnstructuredFileLoader):
    """Load `CSV` files using `Unstructured`.
    Like other
    Unstructured loaders, UnstructuredCSVLoader can be used in both
    "single" and "elements" mode. If you use the loader in "elements"
    mode, the CSV file will be a single Unstructured Table element.
    If you use the loader in "elements" mode, an HTML representation
    of the table will be available in the "text_as_html" key in the
    document metadata.
    Examples
    --------
    from langchain_community.document_loaders.csv_loader import UnstructuredCSVLoader
    loader = UnstructuredCSVLoader("stanley-cups.csv", mode="elements")
loader = UnstructuredCSVLoader("stanley-cups.csv", mode="elements")
    docs = loader.load()
    """
[docs]    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """
        Args:
            file_path: The path to the CSV file.
            mode: The mode to use when loading the CSV file.
              Optional. Defaults to "single".
            **unstructured_kwargs: Keyword arguments to pass to unstructured.
        """
        validate_unstructured_version(min_unstructured_version="0.6.8")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.csv import partition_csv
        return partition_csv(filename=self.file_path, **self.unstructured_kwargs)
Source code for langchain_community.document_loaders.pyspark_dataframe
import itertools
import logging
import sys
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Tuple
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__file__)
if TYPE_CHECKING:
    from pyspark.sql import SparkSession
[docs]class PySparkDataFrameLoader(BaseLoader):
    """Load `PySpark` DataFrames."""
[docs]    def __init__(
        self,
        spark_session: Optional["SparkSession"] = None,
        df: Optional[Any] = None,
        page_content_column: str = "text",
        fraction_of_memory: float = 0.1,
    ):
        """Initialize with a Spark DataFrame object.
        Args:
            spark_session: The SparkSession object.
            df: The Spark DataFrame object.
            page_content_column: The name of the column containing the page content.
             Defaults to "text".
            fraction_of_memory: The fraction of memory to use. Defaults to 0.1.
        """
        try:
            from pyspark.sql import DataFrame, SparkSession
        except ImportError:
            raise ImportError(
                "pyspark is not installed. "
                "Please install it with `pip install pyspark`"
            )
        self.spark = (
            spark_session if spark_session else SparkSession.builder.getOrCreate()
        )
        if not isinstance(df, DataFrame):
            raise ValueError(
                f"Expected data_frame to be a PySpark DataFrame, got {type(df)}"
            )
        self.df = df
        self.page_content_column = page_content_column
        self.fraction_of_memory = fraction_of_memory
self.fraction_of_memory = fraction_of_memory
        self.num_rows, self.max_num_rows = self.get_num_rows()
        self.rdd_df = self.df.rdd.map(list)
        self.column_names = self.df.columns
[docs]    def get_num_rows(self) -> Tuple[int, int]:
        """Gets the number of "feasible" rows for the DataFrame"""
        try:
            import psutil
        except ImportError as e:
            raise ImportError(
                "psutil not installed. Please install it with `pip install psutil`."
            ) from e
        row = self.df.limit(1).collect()[0]
        estimated_row_size = sys.getsizeof(row)
        mem_info = psutil.virtual_memory()
        available_memory = mem_info.available
        max_num_rows = int(
            (available_memory / estimated_row_size) * self.fraction_of_memory
        )
        return min(max_num_rows, self.df.count()), max_num_rows
[docs]    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader for document content."""
        for row in self.rdd_df.toLocalIterator():
            metadata = {self.column_names[i]: row[i] for i in range(len(row))}
            text = metadata[self.page_content_column]
            metadata.pop(self.page_content_column)
            yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load from the dataframe."""
        if self.df.count() > self.max_num_rows:
            logger.warning(
                f"The number of DataFrame rows is {self.df.count()}, "
                f"but we will only include the amount "
                f"of rows that can reasonably fit in memory: {self.num_rows}."
            )
Source code for langchain_community.document_loaders.directory
import concurrent
import logging
import random
from pathlib import Path
from typing import Any, List, Optional, Type, Union
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
from langchain_community.document_loaders.html_bs import BSHTMLLoader
from langchain_community.document_loaders.text import TextLoader
from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
FILE_LOADER_TYPE = Union[
    Type[UnstructuredFileLoader], Type[TextLoader], Type[BSHTMLLoader]
]
logger = logging.getLogger(__name__)
def _is_visible(p: Path) -> bool:
    parts = p.parts
    for _p in parts:
        if _p.startswith("."):
            return False
    return True
[docs]class DirectoryLoader(BaseLoader):
    """Load from a directory."""
[docs]    def __init__(
        self,
        path: str,
        glob: str = "**/[!.]*",
        silent_errors: bool = False,
        load_hidden: bool = False,
        loader_cls: FILE_LOADER_TYPE = UnstructuredFileLoader,
        loader_kwargs: Union[dict, None] = None,
        recursive: bool = False,
        show_progress: bool = False,
        use_multithreading: bool = False,
        max_concurrency: int = 4,
        *,
        sample_size: int = 0,
        randomize_sample: bool = False,
        sample_seed: Union[int, None] = None,
    ):
        """Initialize with a path to directory and how to glob over it.
        Args:
            path: Path to directory.
            glob: Glob pattern to use to find files. Defaults to "**/[!.]*"