viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.chat_loaders.whatsapp
import logging
import os
import re
import zipfile
from typing import Iterator, List, Union
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import AIMessage, HumanMessage
from langchain_community.chat_loaders.base import BaseChatLoader
logger = logging.getLogger(__name__)
[docs]class WhatsAppChatLoader(BaseChatLoader):
    """Load `WhatsApp` conversations from a dump zip file or directory."""
[docs]    def __init__(self, path: str):
        """Initialize the WhatsAppChatLoader.
        Args:
            path (str): Path to the exported WhatsApp chat
                zip directory, folder, or file.
        To generate the dump, open the chat, click the three dots in the top
        right corner, and select "More". Then select "Export chat" and
        choose "Without media".
        """
        self.path = path
        ignore_lines = [
            "This message was deleted",
            "<Media omitted>",
            "image omitted",
            "Messages and calls are end-to-end encrypted. No one outside of this chat,"
            " not even WhatsApp, can read or listen to them.",
        ]
        self._ignore_lines = re.compile(
            r"(" + "|".join([r"\u200E*" + line for line in ignore_lines]) + r")",
            flags=re.IGNORECASE,
        )
        self._message_line_regex = re.compile(
            r"\u200E*\[?(\d{1,2}/\d{1,2}/\d{2,4}, \d{1,2}:\d{2}:\d{2} (?:AM|PM))\]?[ \u200E]*([^:]+): (.+)",  # noqa
flags=re.IGNORECASE,
        )
    def _load_single_chat_session(self, file_path: str) -> ChatSession:
        """Load a single chat session from a file.
        Args:
            file_path (str): Path to the chat file.
        Returns:
            ChatSession: The loaded chat session.
        """
        with open(file_path, "r", encoding="utf-8") as file:
            txt = file.read()
        # Split messages by newlines, but keep multi-line messages grouped
        chat_lines: List[str] = []
        current_message = ""
        for line in txt.split("\n"):
            if self._message_line_regex.match(line):
                if current_message:
                    chat_lines.append(current_message)
                current_message = line
            else:
                current_message += " " + line.strip()
        if current_message:
            chat_lines.append(current_message)
        results: List[Union[HumanMessage, AIMessage]] = []
        for line in chat_lines:
            result = self._message_line_regex.match(line.strip())
            if result:
                timestamp, sender, text = result.groups()
                if not self._ignore_lines.match(text.strip()):
                    results.append(
                        HumanMessage(
                            role=sender,
                            content=text,
                            additional_kwargs={
                                "sender": sender,
                                "events": [{"message_time": timestamp}],
                            },
                        )
                    )
            else:
                logger.debug(f"Could not parse line: {line}")
        return ChatSession(messages=results)
    def _iterate_files(self, path: str) -> Iterator[str]:
        """Iterate over the files in a directory or zip file.
        Args:
            path (str): Path to the directory or zip file.
Args:
            path (str): Path to the directory or zip file.
        Yields:
            str: The path to each file.
        """
        if os.path.isfile(path):
            yield path
        elif os.path.isdir(path):
            for root, _, files in os.walk(path):
                for file in files:
                    if file.endswith(".txt"):
                        yield os.path.join(root, file)
        elif zipfile.is_zipfile(path):
            with zipfile.ZipFile(path) as zip_file:
                for file in zip_file.namelist():
                    if file.endswith(".txt"):
                        yield zip_file.extract(file)
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy load the messages from the chat file and yield
        them as chat sessions.
        Yields:
            Iterator[ChatSession]: The loaded chat sessions.
        """
        yield self._load_single_chat_session(self.path)
Source code for langchain_community.chat_loaders.base
from abc import ABC, abstractmethod
from typing import Iterator, List
from langchain_core.chat_sessions import ChatSession
[docs]class BaseChatLoader(ABC):
    """Base class for chat loaders."""
[docs]    @abstractmethod
    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy load the chat sessions."""
[docs]    def load(self) -> List[ChatSession]:
        """Eagerly load the chat sessions into memory."""
        return list(self.lazy_load())
Source code for langchain_community.chat_loaders.imessage
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, Iterator, List, Optional, Union
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import HumanMessage
from langchain_community.chat_loaders.base import BaseChatLoader
if TYPE_CHECKING:
    import sqlite3
[docs]class IMessageChatLoader(BaseChatLoader):
    """Load chat sessions from the `iMessage` chat.db SQLite file.
    It only works on macOS when you have iMessage enabled and have the chat.db file.
    The chat.db file is likely located at ~/Library/Messages/chat.db. However, your
    terminal may not have permission to access this file. To resolve this, you can
    copy the file to a different location, change the permissions of the file, or
    grant full disk access for your terminal emulator
    in System Settings > Security and Privacy > Full Disk Access.
    """
[docs]    def __init__(self, path: Optional[Union[str, Path]] = None):
        """
        Initialize the IMessageChatLoader.
        Args:
            path (str or Path, optional): Path to the chat.db SQLite file.
                Defaults to None, in which case the default path
                ~/Library/Messages/chat.db will be used.
        """
        if path is None:
            path = Path.home() / "Library" / "Messages" / "chat.db"
        self.db_path = path if isinstance(path, Path) else Path(path)
        if not self.db_path.exists():
            raise FileNotFoundError(f"File {self.db_path} not found")
        try:
            import sqlite3  # noqa: F401
        except ImportError as e:
            raise ImportError(
except ImportError as e:
            raise ImportError(
                "The sqlite3 module is required to load iMessage chats.\n"
                "Please install it with `pip install pysqlite3`"
            ) from e
    def _parse_attributedBody(self, attributedBody: bytes) -> str:
        """
        Parse the attributedBody field of the message table
        for the text content of the message.
        The attributedBody field is a binary blob that contains
        the message content after the byte string b"NSString":
                              5 bytes      1-3 bytes    `len` bytes
        ... | b"NSString" |   preamble   |   `len`   |    contents    | ...
        The 5 preamble bytes are always b"\x01\x94\x84\x01+"
        The size of `len` is either 1 byte or 3 bytes:
        - If the first byte in `len` is b"\x81" then `len` is 3 bytes long.
          So the message length is the 2 bytes after, in little Endian.
        - Otherwise, the size of `len` is 1 byte, and the message length is
          that byte.
        Args:
            attributedBody (bytes): attributedBody field of the message table.
        Return:
            str: Text content of the message.
        """
        content = attributedBody.split(b"NSString")[1][5:]
        length, start = content[0], 1
        if content[0] == 129:
            length, start = int.from_bytes(content[1:3], "little"), 3
        return content[start : start + length].decode("utf-8", errors="ignore")
    def _load_single_chat_session(
def _load_single_chat_session(
        self, cursor: "sqlite3.Cursor", chat_id: int
    ) -> ChatSession:
        """
        Load a single chat session from the iMessage chat.db.
        Args:
            cursor: SQLite cursor object.
            chat_id (int): ID of the chat session to load.
        Returns:
            ChatSession: Loaded chat session.
        """
        results: List[HumanMessage] = []
        query = """
        SELECT message.date, handle.id, message.text, message.attributedBody
        FROM message
        JOIN chat_message_join ON message.ROWID = chat_message_join.message_id
        JOIN handle ON message.handle_id = handle.ROWID
        WHERE chat_message_join.chat_id = ?
        ORDER BY message.date ASC;
        """
        cursor.execute(query, (chat_id,))
        messages = cursor.fetchall()
        for date, sender, text, attributedBody in messages:
            if text:
                content = text
            elif attributedBody:
                content = self._parse_attributedBody(attributedBody)
            else:  # Skip messages with no content
                continue
            results.append(
                HumanMessage(
                    role=sender,
                    content=content,
                    additional_kwargs={
                        "message_time": date,
                        "sender": sender,
                    },
                )
            )
        return ChatSession(messages=results)
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """
        Lazy load the chat sessions from the iMessage chat.db
        and yield them in the required format.
        Yields:
            ChatSession: Loaded chat session.
        """
        import sqlite3
        try:
            conn = sqlite3.connect(self.db_path)
import sqlite3
        try:
            conn = sqlite3.connect(self.db_path)
        except sqlite3.OperationalError as e:
            raise ValueError(
                f"Could not open iMessage DB file {self.db_path}.\n"
                "Make sure your terminal emulator has disk access to this file.\n"
                "   You can either copy the DB file to an accessible location"
                " or grant full disk access for your terminal emulator."
                "  You can grant full disk access for your terminal emulator"
                " in System Settings > Security and Privacy > Full Disk Access."
            ) from e
        cursor = conn.cursor()
        # Fetch the list of chat IDs sorted by time (most recent first)
        query = """SELECT chat_id
        FROM message
        JOIN chat_message_join ON message.ROWID = chat_message_join.message_id
        GROUP BY chat_id
        ORDER BY MAX(date) DESC;"""
        cursor.execute(query)
        chat_ids = [row[0] for row in cursor.fetchall()]
        for chat_id in chat_ids:
            yield self._load_single_chat_session(cursor, chat_id)
        conn.close()
Source code for langchain_community.chat_loaders.facebook_messenger
import json
import logging
from pathlib import Path
from typing import Iterator, Union
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import HumanMessage
from langchain_community.chat_loaders.base import BaseChatLoader
logger = logging.getLogger(__file__)
[docs]class SingleFileFacebookMessengerChatLoader(BaseChatLoader):
    """Load `Facebook Messenger` chat data from a single file.
    Args:
        path (Union[Path, str]): The path to the chat file.
    Attributes:
        path (Path): The path to the chat file.
    """
[docs]    def __init__(self, path: Union[Path, str]) -> None:
        super().__init__()
        self.file_path = path if isinstance(path, Path) else Path(path)
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy loads the chat data from the file.
        Yields:
            ChatSession: A chat session containing the loaded messages.
        """
        with open(self.file_path) as f:
            data = json.load(f)
        sorted_data = sorted(data["messages"], key=lambda x: x["timestamp_ms"])
        messages = []
        for m in sorted_data:
            messages.append(
                HumanMessage(
                    content=m["content"], additional_kwargs={"sender": m["sender_name"]}
                )
            )
        yield ChatSession(messages=messages)
[docs]class FolderFacebookMessengerChatLoader(BaseChatLoader):
    """Load `Facebook Messenger` chat data from a folder.
    Args:
        path (Union[str, Path]): The path to the directory
            containing the chat files.
    Attributes:
containing the chat files.
    Attributes:
        path (Path): The path to the directory containing the chat files.
    """
[docs]    def __init__(self, path: Union[str, Path]) -> None:
        super().__init__()
        self.directory_path = Path(path) if isinstance(path, str) else path
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy loads the chat data from the folder.
        Yields:
            ChatSession: A chat session containing the loaded messages.
        """
        inbox_path = self.directory_path / "inbox"
        for _dir in inbox_path.iterdir():
            if _dir.is_dir():
                for _file in _dir.iterdir():
                    if _file.suffix.lower() == ".json":
                        file_loader = SingleFileFacebookMessengerChatLoader(path=_file)
                        for result in file_loader.lazy_load():
                            yield result
Source code for langchain_community.chat_loaders.utils
"""Utilities for chat loaders."""
from copy import deepcopy
from typing import Iterable, Iterator, List
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import AIMessage, BaseMessage
[docs]def merge_chat_runs_in_session(
    chat_session: ChatSession, delimiter: str = "\n\n"
) -> ChatSession:
    """Merge chat runs together in a chat session.
    A chat run is a sequence of messages from the same sender.
    Args:
        chat_session: A chat session.
    Returns:
        A chat session with merged chat runs.
    """
    messages: List[BaseMessage] = []
    for message in chat_session["messages"]:
        if not isinstance(message.content, str):
            raise ValueError(
                "Chat Loaders only support messages with content type string, "
                f"got {message.content}"
            )
        if not messages:
            messages.append(deepcopy(message))
        elif (
            isinstance(message, type(messages[-1]))
            and messages[-1].additional_kwargs.get("sender") is not None
            and messages[-1].additional_kwargs["sender"]
            == message.additional_kwargs.get("sender")
        ):
            if not isinstance(messages[-1].content, str):
                raise ValueError(
                    "Chat Loaders only support messages with content type string, "
                    f"got {messages[-1].content}"
                )
            messages[-1].content = (
                messages[-1].content + delimiter + message.content
            ).strip()
            messages[-1].additional_kwargs.get("events", []).extend(
                message.additional_kwargs.get("events") or []
            )
        else:
            messages.append(deepcopy(message))
)
        else:
            messages.append(deepcopy(message))
    return ChatSession(messages=messages)
[docs]def merge_chat_runs(chat_sessions: Iterable[ChatSession]) -> Iterator[ChatSession]:
    """Merge chat runs together.
    A chat run is a sequence of messages from the same sender.
    Args:
        chat_sessions: A list of chat sessions.
    Returns:
        A list of chat sessions with merged chat runs.
    """
    for chat_session in chat_sessions:
        yield merge_chat_runs_in_session(chat_session)
[docs]def map_ai_messages_in_session(chat_sessions: ChatSession, sender: str) -> ChatSession:
    """Convert messages from the specified 'sender' to AI messages.
    This is useful for fine-tuning the AI to adapt to your voice.
    """
    messages = []
    num_converted = 0
    for message in chat_sessions["messages"]:
        if message.additional_kwargs.get("sender") == sender:
            message = AIMessage(
                content=message.content,
                additional_kwargs=message.additional_kwargs.copy(),
                example=getattr(message, "example", None),
            )
            num_converted += 1
        messages.append(message)
    return ChatSession(messages=messages)
[docs]def map_ai_messages(
    chat_sessions: Iterable[ChatSession], sender: str
) -> Iterator[ChatSession]:
    """Convert messages from the specified 'sender' to AI messages.
    This is useful for fine-tuning the AI to adapt to your voice.
    """
    for chat_session in chat_sessions:
        yield map_ai_messages_in_session(chat_session, sender)
Source code for langchain_community.chat_loaders.slack
import json
import logging
import re
import zipfile
from pathlib import Path
from typing import Dict, Iterator, List, Union
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import AIMessage, HumanMessage
from langchain_community.chat_loaders.base import BaseChatLoader
logger = logging.getLogger(__name__)
[docs]class SlackChatLoader(BaseChatLoader):
    """Load `Slack` conversations from a dump zip file."""
[docs]    def __init__(
        self,
        path: Union[str, Path],
    ):
        """
        Initialize the chat loader with the path to the exported Slack dump zip file.
        :param path: Path to the exported Slack dump zip file.
        """
        self.zip_path = path if isinstance(path, Path) else Path(path)
        if not self.zip_path.exists():
            raise FileNotFoundError(f"File {self.zip_path} not found")
    def _load_single_chat_session(self, messages: List[Dict]) -> ChatSession:
        results: List[Union[AIMessage, HumanMessage]] = []
        previous_sender = None
        for message in messages:
            if not isinstance(message, dict):
                continue
            text = message.get("text", "")
            timestamp = message.get("ts", "")
            sender = message.get("user", "")
            if not sender:
                continue
            skip_pattern = re.compile(
                r"<@U\d+> has joined the channel", flags=re.IGNORECASE
            )
            if skip_pattern.match(text):
                continue
            if sender == previous_sender:
                results[-1].content += "\n\n" + text
                results[-1].additional_kwargs["events"].append(
                    {"message_time": timestamp}
                )
{"message_time": timestamp}
                )
            else:
                results.append(
                    HumanMessage(
                        role=sender,
                        content=text,
                        additional_kwargs={
                            "sender": sender,
                            "events": [{"message_time": timestamp}],
                        },
                    )
                )
            previous_sender = sender
        return ChatSession(messages=results)
    def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]:
        """Read JSON data from a zip subfile."""
        with zip_file.open(file_path, "r") as f:
            data = json.load(f)
        if not isinstance(data, list):
            raise ValueError(f"Expected list of dictionaries, got {type(data)}")
        return data
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """
        Lazy load the chat sessions from the Slack dump file and yield them
        in the required format.
        :return: Iterator of chat sessions containing messages.
        """
        with zipfile.ZipFile(str(self.zip_path), "r") as zip_file:
            for file_path in zip_file.namelist():
                if file_path.endswith(".json"):
                    messages = self._read_json(zip_file, file_path)
                    yield self._load_single_chat_session(messages)
Source code for langchain_community.chat_loaders.telegram
import json
import logging
import os
import tempfile
import zipfile
from pathlib import Path
from typing import Iterator, List, Union
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
from langchain_community.chat_loaders.base import BaseChatLoader
logger = logging.getLogger(__name__)
[docs]class TelegramChatLoader(BaseChatLoader):
    """Load `telegram` conversations to LangChain chat messages.
    To export, use the Telegram Desktop app from
    https://desktop.telegram.org/, select a conversation, click the three dots
    in the top right corner, and select "Export chat history". Then select
    "Machine-readable JSON" (preferred) to export. Note: the 'lite' versions of
    the desktop app (like "Telegram for MacOS") do not support exporting chat
    history.
    """
[docs]    def __init__(
        self,
        path: Union[str, Path],
    ):
        """Initialize the TelegramChatLoader.
        Args:
            path (Union[str, Path]): Path to the exported Telegram chat zip,
                 directory, json, or HTML file.
        """
        self.path = path if isinstance(path, str) else str(path)
    def _load_single_chat_session_html(self, file_path: str) -> ChatSession:
        """Load a single chat session from an HTML file.
        Args:
            file_path (str): Path to the HTML file.
        Returns:
            ChatSession: The loaded chat session.
        """
        try:
            from bs4 import BeautifulSoup
        except ImportError:
            raise ImportError(
                "Please install the 'beautifulsoup4' package to load"
raise ImportError(
                "Please install the 'beautifulsoup4' package to load"
                " Telegram HTML files. You can do this by running"
                "'pip install beautifulsoup4' in your terminal."
            )
        with open(file_path, "r", encoding="utf-8") as file:
            soup = BeautifulSoup(file, "html.parser")
        results: List[Union[HumanMessage, AIMessage]] = []
        previous_sender = None
        for message in soup.select(".message.default"):
            timestamp = message.select_one(".pull_right.date.details")["title"]
            from_name_element = message.select_one(".from_name")
            if from_name_element is None and previous_sender is None:
                logger.debug("from_name not found in message")
                continue
            elif from_name_element is None:
                from_name = previous_sender
            else:
                from_name = from_name_element.text.strip()
            text = message.select_one(".text").text.strip()
            results.append(
                HumanMessage(
                    content=text,
                    additional_kwargs={
                        "sender": from_name,
                        "events": [{"message_time": timestamp}],
                    },
                )
            )
            previous_sender = from_name
        return ChatSession(messages=results)
    def _load_single_chat_session_json(self, file_path: str) -> ChatSession:
        """Load a single chat session from a JSON file.
        Args:
            file_path (str): Path to the JSON file.
        Returns:
            ChatSession: The loaded chat session.
        """
        with open(file_path, "r", encoding="utf-8") as file:
            data = json.load(file)
        messages = data.get("messages", [])
        results: List[BaseMessage] = []