Tony20100/langchain_full huggingface dataset:
Source code for langchain_community.agent_toolkits.gmail.toolkit
from __future__ import annotations
from typing import TYPE_CHECKING, List
from langchain_core.pydantic_v1 import Field
from langchain_core.tools import BaseToolkit
from langchain_community.tools import BaseTool
from langchain_community.tools.gmail.create_draft import GmailCreateDraft
from langchain_community.tools.gmail.get_message import GmailGetMessage
from langchain_community.tools.gmail.get_thread import GmailGetThread
from langchain_community.tools.gmail.search import GmailSearch
from langchain_community.tools.gmail.send_message import GmailSendMessage
from langchain_community.tools.gmail.utils import build_resource_service
if TYPE_CHECKING:
    # This is for linting and IDE typehints
    from googleapiclient.discovery import Resource
else:
    try:
        # We do this so pydantic can resolve the types when instantiating
        from googleapiclient.discovery import Resource
    except ImportError:
        pass
SCOPES = ["https://mail.google.com/"]
[docs]class GmailToolkit(BaseToolkit):
    """Toolkit for interacting with Gmail.
    *Security Note*: This toolkit contains tools that can read and modify
        the state of a service; e.g., by reading, creating, updating, deleting
        data associated with this service.
        For example, this toolkit can be used to send emails on behalf of the
        associated account.
        See https://python.langchain.com/docs/security for more information.
    """
    api_resource: Resource = Field(default_factory=build_resource_service)
    class Config:
        """Pydantic config."""
        arbitrary_types_allowed = True
[docs]    def get_tools(self) -> List[BaseTool]:
        """Get the tools in the toolkit."""
        return [
            GmailCreateDraft(api_resource=self.api_resource),
Source code for langchain_community.agent_toolkits.polygon.toolkit
from typing import List
from langchain_core.tools import BaseToolkit
from langchain_community.tools import BaseTool
from langchain_community.tools.polygon import (
    PolygonAggregates,
    PolygonFinancials,
    PolygonLastQuote,
    PolygonTickerNews,
)
from langchain_community.utilities.polygon import PolygonAPIWrapper
[docs]class PolygonToolkit(BaseToolkit):
    """Polygon Toolkit."""
    tools: List[BaseTool] = []
[docs]    @classmethod
    def from_polygon_api_wrapper(
        cls, polygon_api_wrapper: PolygonAPIWrapper
    ) -> "PolygonToolkit":
        tools = [
            PolygonAggregates(
                api_wrapper=polygon_api_wrapper,
            ),
            PolygonLastQuote(
                api_wrapper=polygon_api_wrapper,
            ),
            PolygonTickerNews(
                api_wrapper=polygon_api_wrapper,
            ),
            PolygonFinancials(
                api_wrapper=polygon_api_wrapper,
            ),
        ]
        return cls(tools=tools)
[docs]    def get_tools(self) -> List[BaseTool]:
        """Get the tools in the toolkit."""
        return self.tools
Source code for langchain_community.agent_toolkits.steam.toolkit
"""Steam Toolkit."""
from typing import List
from langchain_core.tools import BaseToolkit
from langchain_community.tools import BaseTool
from langchain_community.tools.steam.prompt import (
    STEAM_GET_GAMES_DETAILS,
    STEAM_GET_RECOMMENDED_GAMES,
)
from langchain_community.tools.steam.tool import SteamWebAPIQueryRun
from langchain_community.utilities.steam import SteamWebAPIWrapper
[docs]class SteamToolkit(BaseToolkit):
    """Steam Toolkit."""
    tools: List[BaseTool] = []
[docs]    @classmethod
    def from_steam_api_wrapper(
        cls, steam_api_wrapper: SteamWebAPIWrapper
    ) -> "SteamToolkit":
        operations: List[dict] = [
            {
                "mode": "get_games_details",
                "name": "Get Games Details",
                "description": STEAM_GET_GAMES_DETAILS,
            },
            {
                "mode": "get_recommended_games",
                "name": "Get Recommended Games",
                "description": STEAM_GET_RECOMMENDED_GAMES,
            },
        ]
        tools = [
            SteamWebAPIQueryRun(
                name=action["name"],
                description=action["description"],
                mode=action["mode"],
                api_wrapper=steam_api_wrapper,
            )
            for action in operations
        ]
        return cls(tools=tools)  # type: ignore[arg-type]
[docs]    def get_tools(self) -> List[BaseTool]:
        """Get the tools in the toolkit."""
        return self.tools
Source code for langchain_community.chat_loaders.slack
import json
import logging
import re
import zipfile
from pathlib import Path
from typing import Dict, Iterator, List, Union
from langchain_core.chat_loaders import BaseChatLoader
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import AIMessage, HumanMessage
logger = logging.getLogger(__name__)
[docs]class SlackChatLoader(BaseChatLoader):
    """Load `Slack` conversations from a dump zip file."""
[docs]    def __init__(
        self,
        path: Union[str, Path],
    ):
        """
        Initialize the chat loader with the path to the exported Slack dump zip file.
        :param path: Path to the exported Slack dump zip file.
        """
        self.zip_path = path if isinstance(path, Path) else Path(path)
        if not self.zip_path.exists():
            raise FileNotFoundError(f"File {self.zip_path} not found")
    def _load_single_chat_session(self, messages: List[Dict]) -> ChatSession:
        results: List[Union[AIMessage, HumanMessage]] = []
        previous_sender = None
        for message in messages:
            if not isinstance(message, dict):
                continue
            text = message.get("text", "")
            timestamp = message.get("ts", "")
            sender = message.get("user", "")
            if not sender:
                continue
            skip_pattern = re.compile(
                r"<@U\d+> has joined the channel", flags=re.IGNORECASE
            )
            if skip_pattern.match(text):
                continue
            if sender == previous_sender:
                results[-1].content += "\n\n" + text
                results[-1].additional_kwargs["events"].append(
                    {"message_time": timestamp}
                )
{"message_time": timestamp}
                )
            else:
                results.append(
                    HumanMessage(  # type: ignore[call-arg]
                        role=sender,
                        content=text,
                        additional_kwargs={
                            "sender": sender,
                            "events": [{"message_time": timestamp}],
                        },
                    )
                )
            previous_sender = sender
        return ChatSession(messages=results)
    def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]:
        """Read JSON data from a zip subfile."""
        with zip_file.open(file_path, "r") as f:
            data = json.load(f)
        if not isinstance(data, list):
            raise ValueError(f"Expected list of dictionaries, got {type(data)}")
        return data
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """
        Lazy load the chat sessions from the Slack dump file and yield them
        in the required format.
        :return: Iterator of chat sessions containing messages.
        """
        with zipfile.ZipFile(str(self.zip_path), "r") as zip_file:
            for file_path in zip_file.namelist():
                if file_path.endswith(".json"):
                    messages = self._read_json(zip_file, file_path)
                    yield self._load_single_chat_session(messages)
Source code for langchain_community.chat_loaders.gmail
import base64
import re
from typing import Any, Iterator
from langchain_core._api.deprecation import deprecated
from langchain_core.chat_loaders import BaseChatLoader
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import HumanMessage
def _extract_email_content(msg: Any) -> HumanMessage:
    from_email = None
    for values in msg["payload"]["headers"]:
        name = values["name"]
        if name == "From":
            from_email = values["value"]
    if from_email is None:
        raise ValueError
    for part in msg["payload"]["parts"]:
        if part["mimeType"] == "text/plain":
            data = part["body"]["data"]
            data = base64.urlsafe_b64decode(data).decode("utf-8")
            # Regular expression to split the email body at the first
            # occurrence of a line that starts with "On ... wrote:"
            pattern = re.compile(r"\r\nOn .+(\r\n)*wrote:\r\n")
            # Split the email body and extract the first part
            newest_response = re.split(pattern, data)[0]
            message = HumanMessage(
                content=newest_response, additional_kwargs={"sender": from_email}
            )
            return message
    raise ValueError
def _get_message_data(service: Any, message: Any) -> ChatSession:
    msg = service.users().messages().get(userId="me", id=message["id"]).execute()
    message_content = _extract_email_content(msg)
    in_reply_to = None
    email_data = msg["payload"]["headers"]
    for values in email_data:
        name = values["name"]
        if name == "In-Reply-To":
            in_reply_to = values["value"]
in_reply_to = values["value"]
    if in_reply_to is None:
        raise ValueError
    thread_id = msg["threadId"]
    thread = service.users().threads().get(userId="me", id=thread_id).execute()
    messages = thread["messages"]
    response_email = None
    for message in messages:
        email_data = message["payload"]["headers"]
        for values in email_data:
            if values["name"] == "Message-ID":
                message_id = values["value"]
                if message_id == in_reply_to:
                    response_email = message
    if response_email is None:
        raise ValueError
    starter_content = _extract_email_content(response_email)
    return ChatSession(messages=[starter_content, message_content])
[docs]@deprecated(
    since="0.0.32",
    removal="0.3.0",
    alternative_import="langchain_google_community.GMailLoader",
)
class GMailLoader(BaseChatLoader):
    """Load data from `GMail`.
    There are many ways you could want to load data from GMail.
    This loader is currently fairly opinionated in how to do so.
    The way it does it is it first looks for all messages that you have sent.
    It then looks for messages where you are responding to a previous email.
    It then fetches that previous email, and creates a training example
    of that email, followed by your email.
    Note that there are clear limitations here. For example,
    all examples created are only looking at the previous email for context.
    To use:
    - Set up a Google Developer Account:
        Go to the Google Developer Console, create a project,
        and enable the Gmail API for that project.
and enable the Gmail API for that project.
        This will give you a credentials.json file that you'll need later.
    """
[docs]    def __init__(self, creds: Any, n: int = 100, raise_error: bool = False) -> None:
        super().__init__()
        self.creds = creds
        self.n = n
        self.raise_error = raise_error
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        from googleapiclient.discovery import build
        service = build("gmail", "v1", credentials=self.creds)
        results = (
            service.users()
            .messages()
            .list(userId="me", labelIds=["SENT"], maxResults=self.n)
            .execute()
        )
        messages = results.get("messages", [])
        for message in messages:
            try:
                yield _get_message_data(service, message)
            except Exception as e:
                # TODO: handle errors better
                if self.raise_error:
                    raise e
                else:
                    pass
)
        else:
            messages.append(deepcopy(message))
    return ChatSession(messages=messages)
[docs]def merge_chat_runs(chat_sessions: Iterable[ChatSession]) -> Iterator[ChatSession]:
    """Merge chat runs together.
    A chat run is a sequence of messages from the same sender.
    Args:
        chat_sessions: A list of chat sessions.
    Returns:
        A list of chat sessions with merged chat runs.
    """
    for chat_session in chat_sessions:
        yield merge_chat_runs_in_session(chat_session)
[docs]def map_ai_messages_in_session(chat_sessions: ChatSession, sender: str) -> ChatSession:
    """Convert messages from the specified 'sender' to AI messages.
    This is useful for fine-tuning the AI to adapt to your voice.
    """
    messages = []
    num_converted = 0
    for message in chat_sessions["messages"]:
        if message.additional_kwargs.get("sender") == sender:
            message = AIMessage(
                content=message.content,
                additional_kwargs=message.additional_kwargs.copy(),
                example=getattr(message, "example", None),  # type: ignore[arg-type]
            )
            num_converted += 1
        messages.append(message)
    return ChatSession(messages=messages)
[docs]def map_ai_messages(
    chat_sessions: Iterable[ChatSession], sender: str
) -> Iterator[ChatSession]:
    """Convert messages from the specified 'sender' to AI messages.
    This is useful for fine-tuning the AI to adapt to your voice.
    """
    for chat_session in chat_sessions:
        yield map_ai_messages_in_session(chat_session, sender)
Source code for langchain_community.chat_loaders.telegram
import json
import logging
import os
import tempfile
import zipfile
from pathlib import Path
from typing import Iterator, List, Union
from langchain_core.chat_loaders import BaseChatLoader
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
logger = logging.getLogger(__name__)
[docs]class TelegramChatLoader(BaseChatLoader):
    """Load `telegram` conversations to LangChain chat messages.
    To export, use the Telegram Desktop app from
    https://desktop.telegram.org/, select a conversation, click the three dots
    in the top right corner, and select "Export chat history". Then select
    "Machine-readable JSON" (preferred) to export. Note: the 'lite' versions of
    the desktop app (like "Telegram for MacOS") do not support exporting chat
    history.
    """
[docs]    def __init__(
        self,
        path: Union[str, Path],
    ):
        """Initialize the TelegramChatLoader.
        Args:
            path (Union[str, Path]): Path to the exported Telegram chat zip,
                 directory, json, or HTML file.
        """
        self.path = path if isinstance(path, str) else str(path)
    def _load_single_chat_session_html(self, file_path: str) -> ChatSession:
        """Load a single chat session from an HTML file.
        Args:
            file_path (str): Path to the HTML file.
        Returns:
            ChatSession: The loaded chat session.
        """
        try:
            from bs4 import BeautifulSoup
        except ImportError:
            raise ImportError(
                "Please install the 'beautifulsoup4' package to load"
                " Telegram HTML files. You can do this by running"
" Telegram HTML files. You can do this by running"
                "'pip install beautifulsoup4' in your terminal."
            )
        with open(file_path, "r", encoding="utf-8") as file:
            soup = BeautifulSoup(file, "html.parser")
        results: List[Union[HumanMessage, AIMessage]] = []
        previous_sender = None
        for message in soup.select(".message.default"):
            timestamp = message.select_one(".pull_right.date.details")["title"]
            from_name_element = message.select_one(".from_name")
            if from_name_element is None and previous_sender is None:
                logger.debug("from_name not found in message")
                continue
            elif from_name_element is None:
                from_name = previous_sender
            else:
                from_name = from_name_element.text.strip()
            text = message.select_one(".text").text.strip()
            results.append(
                HumanMessage(
                    content=text,
                    additional_kwargs={
                        "sender": from_name,
                        "events": [{"message_time": timestamp}],
                    },
                )
            )
            previous_sender = from_name
        return ChatSession(messages=results)
    def _load_single_chat_session_json(self, file_path: str) -> ChatSession:
        """Load a single chat session from a JSON file.
        Args:
            file_path (str): Path to the JSON file.
        Returns:
            ChatSession: The loaded chat session.
        """
        with open(file_path, "r", encoding="utf-8") as file:
            data = json.load(file)
        messages = data.get("messages", [])
        results: List[BaseMessage] = []
        for message in messages:
            text = message.get("text", "")
for message in messages:
            text = message.get("text", "")
            timestamp = message.get("date", "")
            from_name = message.get("from", "")
            results.append(
                HumanMessage(
                    content=text,
                    additional_kwargs={
                        "sender": from_name,
                        "events": [{"message_time": timestamp}],
                    },
                )
            )
        return ChatSession(messages=results)
    def _iterate_files(self, path: str) -> Iterator[str]:
        """Iterate over files in a directory or zip file.
        Args:
            path (str): Path to the directory or zip file.
        Yields:
            str: Path to each file.
        """
        if os.path.isfile(path) and path.endswith((".html", ".json")):
            yield path
        elif os.path.isdir(path):
            for root, _, files in os.walk(path):
                for file in files:
                    if file.endswith((".html", ".json")):
                        yield os.path.join(root, file)
        elif zipfile.is_zipfile(path):
            with zipfile.ZipFile(path) as zip_file:
                for file in zip_file.namelist():
                    if file.endswith((".html", ".json")):
                        with tempfile.TemporaryDirectory() as temp_dir:
                            yield zip_file.extract(file, path=temp_dir)
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy load the messages from the chat file and yield them
        in as chat sessions.
        Yields:
            ChatSession: The loaded chat session.
        """
        for file_path in self._iterate_files(self.path):
            if file_path.endswith(".html"):
                yield self._load_single_chat_session_html(file_path)
Source code for langchain_community.chat_loaders.langsmith
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, Optional, Union, cast
from langchain_core.chat_loaders import BaseChatLoader
from langchain_core.chat_sessions import ChatSession
from langchain_core.load.load import load
if TYPE_CHECKING:
    from langsmith.client import Client
    from langsmith.schemas import Run
logger = logging.getLogger(__name__)
[docs]class LangSmithRunChatLoader(BaseChatLoader):
    """
    Load chat sessions from a list of LangSmith "llm" runs.
    Attributes:
        runs (Iterable[Union[str, Run]]): The list of LLM run IDs or run objects.
        client (Client): Instance of LangSmith client for fetching data.
    """
[docs]    def __init__(
        self, runs: Iterable[Union[str, Run]], client: Optional["Client"] = None
    ):
        """
        Initialize a new LangSmithRunChatLoader instance.
        :param runs: List of LLM run IDs or run objects.
        :param client: An instance of LangSmith client, if not provided,
            a new client instance will be created.
        """
        from langsmith.client import Client
        self.runs = runs
        self.client = client or Client()
    def _load_single_chat_session(self, llm_run: "Run") -> ChatSession:
        """
        Convert an individual LangSmith LLM run to a ChatSession.
        :param llm_run: The LLM run object.
        :return: A chat session representing the run's data.
        """
        chat_session = LangSmithRunChatLoader._get_messages_from_llm_run(llm_run)
"Please install it with `pip install langsmith`"
            ) from e
        self.dataset_name = dataset_name
        self.client = client or Client()
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """
        Lazy load the chat sessions from the specified LangSmith dataset.
        This method fetches the chat data from the dataset and
        converts each data point to chat sessions on-the-fly,
        yielding one session at a time.
        :return: Iterator of chat sessions containing messages.
        """
        from langchain_community.adapters import openai as oai_adapter
        data = self.client.read_dataset_openai_finetuning(
            dataset_name=self.dataset_name
        )
        for data_point in data:
            yield ChatSession(
                messages=[
                    oai_adapter.convert_dict_to_message(m)
                    for m in data_point.get("messages", [])
                ],
                functions=data_point.get("functions"),
            )
Source code for langchain_community.chat_loaders.facebook_messenger
import json
import logging
from pathlib import Path
from typing import Iterator, Union
from langchain_core.chat_loaders import BaseChatLoader
from langchain_core.chat_sessions import ChatSession
from langchain_core.messages import HumanMessage
logger = logging.getLogger(__file__)
[docs]class SingleFileFacebookMessengerChatLoader(BaseChatLoader):
    """Load `Facebook Messenger` chat data from a single file.
    Args:
        path (Union[Path, str]): The path to the chat file.
    Attributes:
        path (Path): The path to the chat file.
    """
[docs]    def __init__(self, path: Union[Path, str]) -> None:
        super().__init__()
        self.file_path = path if isinstance(path, Path) else Path(path)
[docs]    def lazy_load(self) -> Iterator[ChatSession]:
        """Lazy loads the chat data from the file.
        Yields:
            ChatSession: A chat session containing the loaded messages.
        """
        with open(self.file_path) as f:
            data = json.load(f)
        sorted_data = sorted(data["messages"], key=lambda x: x["timestamp_ms"])
        messages = []
        for index, m in enumerate(sorted_data):
            if "content" not in m:
                logger.info(
                    f"""Skipping Message No.
                    {index+1} as no content is present in the message"""
                )
                continue
            messages.append(
                HumanMessage(
                    content=m["content"], additional_kwargs={"sender": m["sender_name"]}
                )
            )
        yield ChatSession(messages=messages)
[docs]class FolderFacebookMessengerChatLoader(BaseChatLoader):