viruPVS/langchain-docs huggingface dataset:
of pages with each request. We have to manually check if there
        are more docs based on the length of the returned list of pages, rather than
        just checking for the presence of a `next` key in the response like this page
        would have you do:
        https://developer.atlassian.com/server/confluence/pagination-in-the-rest-api/
        :param retrieval_method: Function used to retrieve docs
        :type retrieval_method: callable
        :return: List of documents
        :rtype: List
        """
        max_pages = kwargs.pop("max_pages")
        docs: List[dict] = []
        while len(docs) < max_pages:
            get_pages = retry(
                reraise=True,
                stop=stop_after_attempt(
                    self.number_of_retries  # type: ignore[arg-type]
                ),
                wait=wait_exponential(
                    multiplier=1,
                    min=self.min_retry_seconds,  # type: ignore[arg-type]
                    max=self.max_retry_seconds,  # type: ignore[arg-type]
                ),
                before_sleep=before_sleep_log(logger, logging.WARNING),
            )(retrieval_method)
            batch = get_pages(**kwargs, start=len(docs))
            if not batch:
                break
            docs.extend(batch)
        return docs[:max_pages]
[docs]    def is_public_page(self, page: dict) -> bool:
        """Check if a page is publicly accessible."""
        restrictions = self.confluence.get_all_restrictions_for_content(page["id"])
        return (
            page["status"] == "current"
            and not restrictions["read"]["restrictions"]["user"]["results"]
            and not restrictions["read"]["restrictions"]["group"]["results"]
        )
[docs]    def process_pages(
        self,
)
[docs]    def process_pages(
        self,
        pages: List[dict],
        include_restricted_content: bool,
        include_attachments: bool,
        include_comments: bool,
        content_format: ContentFormat,
        ocr_languages: Optional[str] = None,
        keep_markdown_format: Optional[bool] = False,
        keep_newlines: bool = False,
    ) -> List[Document]:
        """Process a list of pages into a list of documents."""
        docs = []
        for page in pages:
            if not include_restricted_content and not self.is_public_page(page):
                continue
            doc = self.process_page(
                page,
                include_attachments,
                include_comments,
                content_format,
                ocr_languages=ocr_languages,
                keep_markdown_format=keep_markdown_format,
                keep_newlines=keep_newlines,
            )
            docs.append(doc)
        return docs
[docs]    def process_page(
        self,
        page: dict,
        include_attachments: bool,
        include_comments: bool,
        content_format: ContentFormat,
        ocr_languages: Optional[str] = None,
        keep_markdown_format: Optional[bool] = False,
        keep_newlines: bool = False,
    ) -> Document:
        if keep_markdown_format:
            try:
                from markdownify import markdownify
            except ImportError:
                raise ImportError(
                    "`markdownify` package not found, please run "
                    "`pip install markdownify`"
                )
        if include_comments or not keep_markdown_format:
            try:
                from bs4 import BeautifulSoup  # type: ignore
            except ImportError:
                raise ImportError(
except ImportError:
                raise ImportError(
                    "`beautifulsoup4` package not found, please run "
                    "`pip install beautifulsoup4`"
                )
        if include_attachments:
            attachment_texts = self.process_attachment(page["id"], ocr_languages)
        else:
            attachment_texts = []
        content = content_format.get_content(page)
        if keep_markdown_format:
            # Use markdownify to keep the page Markdown style
            text = markdownify(content, heading_style="ATX") + "".join(attachment_texts)
        else:
            if keep_newlines:
                text = BeautifulSoup(
                    content.replace("</p>", "\n</p>").replace("<br />", "\n"), "lxml"
                ).get_text(" ") + "".join(attachment_texts)
            else:
                text = BeautifulSoup(content, "lxml").get_text(
                    " ", strip=True
                ) + "".join(attachment_texts)
        if include_comments:
            comments = self.confluence.get_page_comments(
                page["id"], expand="body.view.value", depth="all"
            )["results"]
            comment_texts = [
                BeautifulSoup(comment["body"]["view"]["value"], "lxml").get_text(
                    " ", strip=True
                )
                for comment in comments
            ]
            text = text + "".join(comment_texts)
        metadata = {
            "title": page["title"],
            "id": page["id"],
            "source": self.base_url.strip("/") + page["_links"]["webui"],
        }
        if "version" in page and "when" in page["version"]:
            metadata["when"] = page["version"]["when"]
        return Document(
            page_content=text,
            metadata=metadata,
        )
return Document(
            page_content=text,
            metadata=metadata,
        )
[docs]    def process_attachment(
        self,
        page_id: str,
        ocr_languages: Optional[str] = None,
    ) -> List[str]:
        try:
            from PIL import Image  # noqa: F401
        except ImportError:
            raise ImportError(
                "`Pillow` package not found, " "please run `pip install Pillow`"
            )
        # depending on setup you may also need to set the correct path for
        # poppler and tesseract
        attachments = self.confluence.get_attachments_from_content(page_id)["results"]
        texts = []
        for attachment in attachments:
            media_type = attachment["metadata"]["mediaType"]
            absolute_url = self.base_url + attachment["_links"]["download"]
            title = attachment["title"]
            try:
                if media_type == "application/pdf":
                    text = title + self.process_pdf(absolute_url, ocr_languages)
                elif (
                    media_type == "image/png"
                    or media_type == "image/jpg"
                    or media_type == "image/jpeg"
                ):
                    text = title + self.process_image(absolute_url, ocr_languages)
                elif (
                    media_type == "application/vnd.openxmlformats-officedocument"
                    ".wordprocessingml.document"
                ):
                    text = title + self.process_doc(absolute_url)
                elif media_type == "application/vnd.ms-excel":
                    text = title + self.process_xls(absolute_url)
                elif media_type == "image/svg+xml":
                    text = title + self.process_svg(absolute_url, ocr_languages)
                else:
                    continue
                texts.append(text)
            except requests.HTTPError as e:
continue
                texts.append(text)
            except requests.HTTPError as e:
                if e.response.status_code == 404:
                    print(f"Attachment not found at {absolute_url}")
                    continue
                else:
                    raise
        return texts
[docs]    def process_pdf(
        self,
        link: str,
        ocr_languages: Optional[str] = None,
    ) -> str:
        try:
            import pytesseract  # noqa: F401
            from pdf2image import convert_from_bytes  # noqa: F401
        except ImportError:
            raise ImportError(
                "`pytesseract` or `pdf2image` package not found, "
                "please run `pip install pytesseract pdf2image`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        try:
            images = convert_from_bytes(response.content)
        except ValueError:
            return text
        for i, image in enumerate(images):
            image_text = pytesseract.image_to_string(image, lang=ocr_languages)
            text += f"Page {i + 1}:\n{image_text}\n\n"
        return text
[docs]    def process_image(
        self,
        link: str,
        ocr_languages: Optional[str] = None,
    ) -> str:
        try:
            import pytesseract  # noqa: F401
            from PIL import Image  # noqa: F401
        except ImportError:
            raise ImportError(
                "`pytesseract` or `Pillow` package not found, "
"`pytesseract` or `Pillow` package not found, "
                "please run `pip install pytesseract Pillow`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        try:
            image = Image.open(BytesIO(response.content))
        except OSError:
            return text
        return pytesseract.image_to_string(image, lang=ocr_languages)
[docs]    def process_doc(self, link: str) -> str:
        try:
            import docx2txt  # noqa: F401
        except ImportError:
            raise ImportError(
                "`docx2txt` package not found, please run `pip install docx2txt`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        file_data = BytesIO(response.content)
        return docx2txt.process(file_data)
[docs]    def process_xls(self, link: str) -> str:
        import io
        import os
        try:
            import xlrd  # noqa: F401
        except ImportError:
            raise ImportError("`xlrd` package not found, please run `pip install xlrd`")
        try:
            import pandas as pd
        except ImportError:
            raise ImportError(
                "`pandas` package not found, please run `pip install pandas`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        filename = os.path.basename(link)
        # Getting the whole content of the url after filename,
        # Example: ".csv?version=2&modificationDate=1631800010678&cacheVersion=1&api=v2"
        file_extension = os.path.splitext(filename)[1]
        if file_extension.startswith(
            ".csv"
        ):  # if the extension found in the url is ".csv"
            content_string = response.content.decode("utf-8")
            df = pd.read_csv(io.StringIO(content_string))
            text += df.to_string(index=False, header=False) + "\n\n"
        else:
            workbook = xlrd.open_workbook(file_contents=response.content)
            for sheet in workbook.sheets():
                text += f"{sheet.name}:\n"
                for row in range(sheet.nrows):
                    for col in range(sheet.ncols):
                        text += f"{sheet.cell_value(row, col)}\t"
                    text += "\n"
                text += "\n"
        return text
[docs]    def process_svg(
        self,
        link: str,
        ocr_languages: Optional[str] = None,
    ) -> str:
        try:
            import pytesseract  # noqa: F401
            from PIL import Image  # noqa: F401
            from reportlab.graphics import renderPM  # noqa: F401
            from svglib.svglib import svg2rlg  # noqa: F401
        except ImportError:
            raise ImportError(
except ImportError:
            raise ImportError(
                "`pytesseract`, `Pillow`, `reportlab` or `svglib` package not found, "
                "please run `pip install pytesseract Pillow reportlab svglib`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        drawing = svg2rlg(BytesIO(response.content))
        img_data = BytesIO()
        renderPM.drawToFile(drawing, img_data, fmt="PNG")
        img_data.seek(0)
        image = Image.open(img_data)
        return pytesseract.image_to_string(image, lang=ocr_languages)
Source code for langchain_community.document_loaders.duckdb_loader
from typing import Dict, List, Optional, cast
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class DuckDBLoader(BaseLoader):
    """Load from `DuckDB`.
    Each document represents one row of the result. The `page_content_columns`
    are written into the `page_content` of the document. The `metadata_columns`
    are written into the `metadata` of the document. By default, all columns
    are written into the `page_content` and none into the `metadata`.
    """
[docs]    def __init__(
        self,
        query: str,
        database: str = ":memory:",
        read_only: bool = False,
        config: Optional[Dict[str, str]] = None,
        page_content_columns: Optional[List[str]] = None,
        metadata_columns: Optional[List[str]] = None,
    ):
        """
        Args:
            query: The query to execute.
            database: The database to connect to. Defaults to ":memory:".
            read_only: Whether to open the database in read-only mode.
              Defaults to False.
            config: A dictionary of configuration options to pass to the database.
              Optional.
            page_content_columns: The columns to write into the `page_content`
              of the document. Optional.
            metadata_columns: The columns to write into the `metadata` of the document.
              Optional.
        """
        self.query = query
        self.database = database
        self.read_only = read_only
        self.config = config or {}
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    def load(self) -> List[Document]:
[docs]    def load(self) -> List[Document]:
        try:
            import duckdb
        except ImportError:
            raise ImportError(
                "Could not import duckdb python package. "
                "Please install it with `pip install duckdb`."
            )
        docs = []
        with duckdb.connect(
            database=self.database, read_only=self.read_only, config=self.config
        ) as con:
            query_result = con.execute(self.query)
            results = query_result.fetchall()
            description = cast(list, query_result.description)
            field_names = [c[0] for c in description]
            if self.page_content_columns is None:
                page_content_columns = field_names
            else:
                page_content_columns = self.page_content_columns
            if self.metadata_columns is None:
                metadata_columns = []
            else:
                metadata_columns = self.metadata_columns
            for result in results:
                page_content = "\n".join(
                    f"{column}: {result[field_names.index(column)]}"
                    for column in page_content_columns
                )
                metadata = {
                    column: result[field_names.index(column)]
                    for column in metadata_columns
                }
                doc = Document(page_content=page_content, metadata=metadata)
                docs.append(doc)
        return docs
Source code for langchain_community.document_loaders.srt
from typing import List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class SRTLoader(BaseLoader):
    """Load `.srt` (subtitle) files."""
[docs]    def __init__(self, file_path: str):
        """Initialize with a file path."""
        try:
            import pysrt  # noqa:F401
        except ImportError:
            raise ImportError(
                "package `pysrt` not found, please install it with `pip install pysrt`"
            )
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load using pysrt file."""
        import pysrt
        parsed_info = pysrt.open(self.file_path)
        text = " ".join([t.text for t in parsed_info])
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
Source code for langchain_community.document_loaders.rspace
import os
from typing import Any, Dict, Iterator, List, Optional, Union
from langchain_core.documents import Document
from langchain_core.utils import get_from_dict_or_env
from langchain_community.document_loaders import PyPDFLoader
from langchain_community.document_loaders.base import BaseLoader
[docs]class RSpaceLoader(BaseLoader):
    """
    Loads  content from RSpace notebooks, folders, documents or PDF Gallery files into
    Langchain documents.
    Maps  RSpace document <-> Langchain Document in 1-1. PDFs are imported using PyPDF.
    Requirements are rspace_client (`pip install rspace_client`) and PyPDF if importing
     PDF docs (`pip install pypdf`).
    """
[docs]    def __init__(
        self, global_id: str, api_key: Optional[str] = None, url: Optional[str] = None
    ):
        """api_key: RSpace API key - can also be supplied as environment variable
        'RSPACE_API_KEY'
        url: str
        The URL of your RSpace instance - can also be supplied as environment
        variable 'RSPACE_URL'
        global_id: str
         The global ID of the resource to load,
        e.g. 'SD12344' (a single document); 'GL12345'(A PDF file in the gallery);
        'NB4567' (a notebook); 'FL12244' (a folder)
        """
        args: Dict[str, Optional[str]] = {
            "api_key": api_key,
            "url": url,
            "global_id": global_id,
        }
        verified_args: Dict[str, str] = RSpaceLoader.validate_environment(args)
        self.api_key = verified_args["api_key"]
self.api_key = verified_args["api_key"]
        self.url = verified_args["url"]
        self.global_id: str = verified_args["global_id"]
[docs]    @classmethod
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that API key and URL exists in environment."""
        values["api_key"] = get_from_dict_or_env(values, "api_key", "RSPACE_API_KEY")
        values["url"] = get_from_dict_or_env(values, "url", "RSPACE_URL")
        if "global_id" not in values or values["global_id"] is None:
            raise ValueError(
                "No value supplied for global_id. Please supply an RSpace global ID"
            )
        return values
    def _create_rspace_client(self) -> Any:
        """Create a RSpace client."""
        try:
            from rspace_client.eln import eln, field_content
        except ImportError:
            raise ImportError("You must run " "`pip install rspace_client`")
        try:
            eln = eln.ELNClient(self.url, self.api_key)
            eln.get_status()
        except Exception:
            raise Exception(
                f"Unable to initialise client - is url {self.url} or "
                f"api key  correct?"
            )
        return eln, field_content.FieldContent
    def _get_doc(self, cli: Any, field_content: Any, d_id: Union[str, int]) -> Document:
        content = ""
        doc = cli.get_document(d_id)
        content += f"<h2>{doc['name']}<h2/>"
        for f in doc["fields"]:
            content += f"{f['name']}\n"
            fc = field_content(f["content"])
fc = field_content(f["content"])
            content += fc.get_text()
            content += "\n"
        return Document(
            metadata={"source": f"rspace: {doc['name']}-{doc['globalId']}"},
            page_content=content,
        )
    def _load_structured_doc(self) -> Iterator[Document]:
        cli, field_content = self._create_rspace_client()
        yield self._get_doc(cli, field_content, self.global_id)
    def _load_folder_tree(self) -> Iterator[Document]:
        cli, field_content = self._create_rspace_client()
        if self.global_id:
            docs_in_folder = cli.list_folder_tree(
                folder_id=self.global_id[2:], typesToInclude=["document"]
            )
        doc_ids: List[int] = [d["id"] for d in docs_in_folder["records"]]
        for doc_id in doc_ids:
            yield self._get_doc(cli, field_content, doc_id)
    def _load_pdf(self) -> Iterator[Document]:
        cli, field_content = self._create_rspace_client()
        file_info = cli.get_file_info(self.global_id)
        _, ext = os.path.splitext(file_info["name"])
        if ext.lower() == ".pdf":
            outfile = f"{self.global_id}.pdf"
            cli.download_file(self.global_id, outfile)
            pdf_loader = PyPDFLoader(outfile)
            for pdf in pdf_loader.lazy_load():
                pdf.metadata["rspace_src"] = self.global_id
                yield pdf
[docs]    def lazy_load(self) -> Iterator[Document]:
        if self.global_id and "GL" in self.global_id:
            for d in self._load_pdf():
                yield d
for d in self._load_pdf():
                yield d
        elif self.global_id and "SD" in self.global_id:
            for d in self._load_structured_doc():
                yield d
        elif self.global_id and self.global_id[0:2] in ["FL", "NB"]:
            for d in self._load_folder_tree():
                yield d
        else:
            raise ValueError("Unknown global ID type")
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
Source code for langchain_community.document_loaders.geodataframe
from typing import Any, Iterator, List
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class GeoDataFrameLoader(BaseLoader):
    """Load `geopandas` Dataframe."""
[docs]    def __init__(self, data_frame: Any, page_content_column: str = "geometry"):
        """Initialize with geopandas Dataframe.
        Args:
            data_frame: geopandas DataFrame object.
            page_content_column: Name of the column containing the page content.
              Defaults to "geometry".
        """
        try:
            import geopandas as gpd
        except ImportError:
            raise ImportError(
                "geopandas package not found, please install it with "
                "`pip install geopandas`"
            )
        if not isinstance(data_frame, gpd.GeoDataFrame):
            raise ValueError(
                f"Expected data_frame to be a gpd.GeoDataFrame, got {type(data_frame)}"
            )
        if page_content_column not in data_frame.columns:
            raise ValueError(
                f"Expected data_frame to have a column named {page_content_column}"
            )
        if not isinstance(data_frame[page_content_column], gpd.GeoSeries):
            raise ValueError(
                f"Expected data_frame[{page_content_column}] to be a GeoSeries"
            )
        self.data_frame = data_frame
        self.page_content_column = page_content_column
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load records from dataframe."""
        # assumes all geometries in GeoSeries are same CRS and Geom Type
        crs_str = self.data_frame.crs.to_string() if self.data_frame.crs else None