viruPVS/langchain-docs huggingface dataset:
values, "azure_cogs_endpoint", "AZURE_COGS_ENDPOINT"
        )
        try:
            import azure.ai.vision as sdk
            values["vision_service"] = sdk.VisionServiceOptions(
                endpoint=azure_cogs_endpoint, key=azure_cogs_key
            )
            values["analysis_options"] = sdk.ImageAnalysisOptions()
            values["analysis_options"].features = (
                sdk.ImageAnalysisFeature.CAPTION
                | sdk.ImageAnalysisFeature.OBJECTS
                | sdk.ImageAnalysisFeature.TAGS
                | sdk.ImageAnalysisFeature.TEXT
            )
        except ImportError:
            raise ImportError(
                "azure-ai-vision is not installed. "
                "Run `pip install azure-ai-vision` to install."
            )
        return values
    def _image_analysis(self, image_path: str) -> Dict:
        try:
            import azure.ai.vision as sdk
        except ImportError:
            pass
        image_src_type = detect_file_src_type(image_path)
        if image_src_type == "local":
            vision_source = sdk.VisionSource(filename=image_path)
        elif image_src_type == "remote":
            vision_source = sdk.VisionSource(url=image_path)
        else:
            raise ValueError(f"Invalid image path: {image_path}")
        image_analyzer = sdk.ImageAnalyzer(
            self.vision_service, vision_source, self.analysis_options
        )
        result = image_analyzer.analyze()
        res_dict = {}
        if result.reason == sdk.ImageAnalysisResultReason.ANALYZED:
            if result.caption is not None:
                res_dict["caption"] = result.caption.content
            if result.objects is not None:
                res_dict["objects"] = [obj.name for obj in result.objects]
res_dict["objects"] = [obj.name for obj in result.objects]
            if result.tags is not None:
                res_dict["tags"] = [tag.name for tag in result.tags]
            if result.text is not None:
                res_dict["text"] = [line.content for line in result.text.lines]
        else:
            error_details = sdk.ImageAnalysisErrorDetails.from_result(result)
            raise RuntimeError(
                f"Image analysis failed.\n"
                f"Reason: {error_details.reason}\n"
                f"Details: {error_details.message}"
            )
        return res_dict
    def _format_image_analysis_result(self, image_analysis_result: Dict) -> str:
        formatted_result = []
        if "caption" in image_analysis_result:
            formatted_result.append("Caption: " + image_analysis_result["caption"])
        if (
            "objects" in image_analysis_result
            and len(image_analysis_result["objects"]) > 0
        ):
            formatted_result.append(
                "Objects: " + ", ".join(image_analysis_result["objects"])
            )
        if "tags" in image_analysis_result and len(image_analysis_result["tags"]) > 0:
            formatted_result.append("Tags: " + ", ".join(image_analysis_result["tags"]))
        if "text" in image_analysis_result and len(image_analysis_result["text"]) > 0:
            formatted_result.append("Text: " + ", ".join(image_analysis_result["text"]))
        return "\n".join(formatted_result)
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        try:
            image_analysis_result = self._image_analysis(query)
Source code for langchain_community.tools.azure_cognitive_services.text_analytics_health
from __future__ import annotations
import logging
from typing import Any, Dict, Optional
from langchain_core.callbacks import CallbackManagerForToolRun
from langchain_core.pydantic_v1 import root_validator
from langchain_core.tools import BaseTool
from langchain_core.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class AzureCogsTextAnalyticsHealthTool(BaseTool):
    """Tool that queries the Azure Cognitive Services Text Analytics for Health API.
    In order to set this up, follow instructions at:
    https://learn.microsoft.com/en-us/azure/ai-services/language-service/text-analytics-for-health/quickstart?tabs=windows&pivots=programming-language-python
    """
    azure_cogs_key: str = ""  #: :meta private:
    azure_cogs_endpoint: str = ""  #: :meta private:
    text_analytics_client: Any  #: :meta private:
    name: str = "azure_cognitive_services_text_analyics_health"
    description: str = (
        "A wrapper around Azure Cognitive Services Text Analytics for Health. "
        "Useful for when you need to identify entities in healthcare data. "
        "Input should be text."
    )
    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and endpoint exists in environment."""
        azure_cogs_key = get_from_dict_or_env(
            values, "azure_cogs_key", "AZURE_COGS_KEY"
        )
        azure_cogs_endpoint = get_from_dict_or_env(
            values, "azure_cogs_endpoint", "AZURE_COGS_ENDPOINT"
        )
        try:
            import azure.ai.textanalytics as sdk
)
        try:
            import azure.ai.textanalytics as sdk
            from azure.core.credentials import AzureKeyCredential
            values["text_analytics_client"] = sdk.TextAnalyticsClient(
                endpoint=azure_cogs_endpoint,
                credential=AzureKeyCredential(azure_cogs_key),
            )
        except ImportError:
            raise ImportError(
                "azure-ai-textanalytics is not installed. "
                "Run `pip install azure-ai-textanalytics` to install."
            )
        return values
    def _text_analysis(self, text: str) -> Dict:
        poller = self.text_analytics_client.begin_analyze_healthcare_entities(
            [{"id": "1", "language": "en", "text": text}]
        )
        result = poller.result()
        res_dict = {}
        docs = [doc for doc in result if not doc.is_error]
        if docs is not None:
            res_dict["entities"] = [
                f"{x.text} is a healthcare entity of type {x.category}"
                for y in docs
                for x in y.entities
            ]
        return res_dict
    def _format_text_analysis_result(self, text_analysis_result: Dict) -> str:
        formatted_result = []
        if "entities" in text_analysis_result:
            formatted_result.append(
                f"""The text contains the following healthcare entities: {
                        ', '.join(text_analysis_result['entities'])
                    }""".replace("\n", " ")
            )
        return "\n".join(formatted_result)
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        try:
Source code for langchain_community.tools.azure_cognitive_services.utils
import os
import tempfile
from urllib.parse import urlparse
import requests
[docs]def detect_file_src_type(file_path: str) -> str:
    """Detect if the file is local or remote."""
    if os.path.isfile(file_path):
        return "local"
    parsed_url = urlparse(file_path)
    if parsed_url.scheme and parsed_url.netloc:
        return "remote"
    return "invalid"
[docs]def download_audio_from_url(audio_url: str) -> str:
    """Download audio from url to local."""
    ext = audio_url.split(".")[-1]
    response = requests.get(audio_url, stream=True)
    response.raise_for_status()
    with tempfile.NamedTemporaryFile(mode="wb", suffix=f".{ext}", delete=False) as f:
        for chunk in response.iter_content(chunk_size=8192):
            f.write(chunk)
    return f.name
Source code for langchain_community.tools.azure_cognitive_services.form_recognizer
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from langchain_core.callbacks import CallbackManagerForToolRun
from langchain_core.pydantic_v1 import root_validator
from langchain_core.tools import BaseTool
from langchain_core.utils import get_from_dict_or_env
from langchain_community.tools.azure_cognitive_services.utils import (
    detect_file_src_type,
)
logger = logging.getLogger(__name__)
[docs]class AzureCogsFormRecognizerTool(BaseTool):
    """Tool that queries the Azure Cognitive Services Form Recognizer API.
    In order to set this up, follow instructions at:
    https://learn.microsoft.com/en-us/azure/applied-ai-services/form-recognizer/quickstarts/get-started-sdks-rest-api?view=form-recog-3.0.0&pivots=programming-language-python
    """
    azure_cogs_key: str = ""  #: :meta private:
    azure_cogs_endpoint: str = ""  #: :meta private:
    doc_analysis_client: Any  #: :meta private:
    name: str = "azure_cognitive_services_form_recognizer"
    description: str = (
        "A wrapper around Azure Cognitive Services Form Recognizer. "
        "Useful for when you need to "
        "extract text, tables, and key-value pairs from documents. "
        "Input should be a url to a document."
    )
    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and endpoint exists in environment."""
        azure_cogs_key = get_from_dict_or_env(
            values, "azure_cogs_key", "AZURE_COGS_KEY"
        )
)
        azure_cogs_endpoint = get_from_dict_or_env(
            values, "azure_cogs_endpoint", "AZURE_COGS_ENDPOINT"
        )
        try:
            from azure.ai.formrecognizer import DocumentAnalysisClient
            from azure.core.credentials import AzureKeyCredential
            values["doc_analysis_client"] = DocumentAnalysisClient(
                endpoint=azure_cogs_endpoint,
                credential=AzureKeyCredential(azure_cogs_key),
            )
        except ImportError:
            raise ImportError(
                "azure-ai-formrecognizer is not installed. "
                "Run `pip install azure-ai-formrecognizer` to install."
            )
        return values
    def _parse_tables(self, tables: List[Any]) -> List[Any]:
        result = []
        for table in tables:
            rc, cc = table.row_count, table.column_count
            _table = [["" for _ in range(cc)] for _ in range(rc)]
            for cell in table.cells:
                _table[cell.row_index][cell.column_index] = cell.content
            result.append(_table)
        return result
    def _parse_kv_pairs(self, kv_pairs: List[Any]) -> List[Any]:
        result = []
        for kv_pair in kv_pairs:
            key = kv_pair.key.content if kv_pair.key else ""
            value = kv_pair.value.content if kv_pair.value else ""
            result.append((key, value))
        return result
    def _document_analysis(self, document_path: str) -> Dict:
        document_src_type = detect_file_src_type(document_path)
        if document_src_type == "local":
            with open(document_path, "rb") as document:
                poller = self.doc_analysis_client.begin_analyze_document(
                    "prebuilt-document", document
                )
"prebuilt-document", document
                )
        elif document_src_type == "remote":
            poller = self.doc_analysis_client.begin_analyze_document_from_url(
                "prebuilt-document", document_path
            )
        else:
            raise ValueError(f"Invalid document path: {document_path}")
        result = poller.result()
        res_dict = {}
        if result.content is not None:
            res_dict["content"] = result.content
        if result.tables is not None:
            res_dict["tables"] = self._parse_tables(result.tables)
        if result.key_value_pairs is not None:
            res_dict["key_value_pairs"] = self._parse_kv_pairs(result.key_value_pairs)
        return res_dict
    def _format_document_analysis_result(self, document_analysis_result: Dict) -> str:
        formatted_result = []
        if "content" in document_analysis_result:
            formatted_result.append(
                f"Content: {document_analysis_result['content']}".replace("\n", " ")
            )
        if "tables" in document_analysis_result:
            for i, table in enumerate(document_analysis_result["tables"]):
                formatted_result.append(f"Table {i}: {table}".replace("\n", " "))
        if "key_value_pairs" in document_analysis_result:
            for kv_pair in document_analysis_result["key_value_pairs"]:
                formatted_result.append(
                    f"{kv_pair[0]}: {kv_pair[1]}".replace("\n", " ")
                )
        return "\n".join(formatted_result)
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        try:
) -> str:
        """Use the tool."""
        try:
            document_analysis_result = self._document_analysis(query)
            if not document_analysis_result:
                return "No good document analysis result was found"
            return self._format_document_analysis_result(document_analysis_result)
        except Exception as e:
            raise RuntimeError(f"Error while running AzureCogsFormRecognizerTool: {e}")
Source code for langchain_community.tools.shell.tool
import asyncio
import platform
import warnings
from typing import Any, List, Optional, Type, Union
from langchain_core.callbacks import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain_core.pydantic_v1 import BaseModel, Field, root_validator
from langchain_core.tools import BaseTool
[docs]class ShellInput(BaseModel):
    """Commands for the Bash Shell tool."""
    commands: Union[str, List[str]] = Field(
        ...,
        description="List of shell commands to run. Deserialized using json.loads",
    )
    """List of shell commands to run."""
    @root_validator
    def _validate_commands(cls, values: dict) -> dict:
        """Validate commands."""
        # TODO: Add real validators
        commands = values.get("commands")
        if not isinstance(commands, list):
            values["commands"] = [commands]
        # Warn that the bash tool is not safe
        warnings.warn(
            "The shell tool has no safeguards by default. Use at your own risk."
        )
        return values
def _get_default_bash_process() -> Any:
    """Get default bash process."""
    try:
        from langchain_experimental.llm_bash.bash import BashProcess
    except ImportError:
        raise ImportError(
            "BashProcess has been moved to langchain experimental."
            "To use this tool, install langchain-experimental "
            "with `pip install langchain-experimental`."
        )
    return BashProcess(return_err_output=True)
def _get_platform() -> str:
    """Get platform."""
    system = platform.system()
    if system == "Darwin":
        return "MacOS"
    return system
if system == "Darwin":
        return "MacOS"
    return system
[docs]class ShellTool(BaseTool):
    """Tool to run shell commands."""
    process: Any = Field(default_factory=_get_default_bash_process)
    """Bash process to run commands."""
    name: str = "terminal"
    """Name of tool."""
    description: str = f"Run shell commands on this {_get_platform()} machine."
    """Description of tool."""
    args_schema: Type[BaseModel] = ShellInput
    """Schema for input arguments."""
    def _run(
        self,
        commands: Union[str, List[str]],
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Run commands and return final output."""
        return self.process.run(commands)
    async def _arun(
        self,
        commands: Union[str, List[str]],
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> str:
        """Run commands asynchronously and return final output."""
        return await asyncio.get_event_loop().run_in_executor(
            None, self.process.run, commands
        )
Source code for langchain_community.document_loaders.obsidian
import functools
import logging
import re
from pathlib import Path
from typing import Any, Dict, List
import yaml
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class ObsidianLoader(BaseLoader):
    """Load `Obsidian` files from directory."""
    FRONT_MATTER_REGEX = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL)
    TEMPLATE_VARIABLE_REGEX = re.compile(r"{{(.*?)}}", re.DOTALL)
    TAG_REGEX = re.compile(r"[^\S\/]#([a-zA-Z_]+[-_/\w]*)")
    DATAVIEW_LINE_REGEX = re.compile(r"^\s*(\w+)::\s*(.*)$", re.MULTILINE)
    DATAVIEW_INLINE_BRACKET_REGEX = re.compile(r"\[(\w+)::\s*(.*)\]", re.MULTILINE)
    DATAVIEW_INLINE_PAREN_REGEX = re.compile(r"\((\w+)::\s*(.*)\)", re.MULTILINE)
[docs]    def __init__(
        self, path: str, encoding: str = "UTF-8", collect_metadata: bool = True
    ):
        """Initialize with a path.
        Args:
            path: Path to the directory containing the Obsidian files.
            encoding: Charset encoding, defaults to "UTF-8"
            collect_metadata: Whether to collect metadata from the front matter.
                Defaults to True.
        """
        self.file_path = path
        self.encoding = encoding
        self.collect_metadata = collect_metadata
    def _replace_template_var(
        self, placeholders: Dict[str, str], match: re.Match
    ) -> str:
) -> str:
        """Replace a template variable with a placeholder."""
        placeholder = f"__TEMPLATE_VAR_{len(placeholders)}__"
        placeholders[placeholder] = match.group(1)
        return placeholder
    def _restore_template_vars(self, obj: Any, placeholders: Dict[str, str]) -> Any:
        """Restore template variables replaced with placeholders to original values."""
        if isinstance(obj, str):
            for placeholder, value in placeholders.items():
                obj = obj.replace(placeholder, f"{{{{{value}}}}}")
        elif isinstance(obj, dict):
            for key, value in obj.items():
                obj[key] = self._restore_template_vars(value, placeholders)
        elif isinstance(obj, list):
            for i, item in enumerate(obj):
                obj[i] = self._restore_template_vars(item, placeholders)
        return obj
    def _parse_front_matter(self, content: str) -> dict:
        """Parse front matter metadata from the content and return it as a dict."""
        if not self.collect_metadata:
            return {}
        match = self.FRONT_MATTER_REGEX.search(content)
        if not match:
            return {}
        placeholders: Dict[str, str] = {}
        replace_template_var = functools.partial(
            self._replace_template_var, placeholders
        )
        front_matter_text = self.TEMPLATE_VARIABLE_REGEX.sub(
            replace_template_var, match.group(1)
        )
        try:
            front_matter = yaml.safe_load(front_matter_text)
            front_matter = self._restore_template_vars(front_matter, placeholders)
            # If tags are a string, split them into a list
            if "tags" in front_matter and isinstance(front_matter["tags"], str):
if "tags" in front_matter and isinstance(front_matter["tags"], str):
                front_matter["tags"] = front_matter["tags"].split(", ")
            return front_matter
        except yaml.parser.ParserError:
            logger.warning("Encountered non-yaml frontmatter")
            return {}
    def _to_langchain_compatible_metadata(self, metadata: dict) -> dict:
        """Convert a dictionary to a compatible with langchain."""
        result = {}
        for key, value in metadata.items():
            if type(value) in {str, int, float}:
                result[key] = value
            else:
                result[key] = str(value)
        return result
    def _parse_document_tags(self, content: str) -> set:
        """Return a set of all tags in within the document."""
        if not self.collect_metadata:
            return set()
        match = self.TAG_REGEX.findall(content)
        if not match:
            return set()
        return {tag for tag in match}
    def _parse_dataview_fields(self, content: str) -> dict:
        """Parse obsidian dataview plugin fields from the content and return it
        as a dict."""
        if not self.collect_metadata:
            return {}
        return {
            **{
                match[0]: match[1]
                for match in self.DATAVIEW_LINE_REGEX.findall(content)
            },
            **{
                match[0]: match[1]
                for match in self.DATAVIEW_INLINE_PAREN_REGEX.findall(content)
            },
            **{
                match[0]: match[1]
                for match in self.DATAVIEW_INLINE_BRACKET_REGEX.findall(content)
            },
        }
    def _remove_front_matter(self, content: str) -> str:
}
    def _remove_front_matter(self, content: str) -> str:
        """Remove front matter metadata from the given content."""
        if not self.collect_metadata:
            return content
        return self.FRONT_MATTER_REGEX.sub("", content)
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        paths = list(Path(self.file_path).glob("**/*.md"))
        docs = []
        for path in paths:
            with open(path, encoding=self.encoding) as f:
                text = f.read()
            front_matter = self._parse_front_matter(text)
            tags = self._parse_document_tags(text)
            dataview_fields = self._parse_dataview_fields(text)
            text = self._remove_front_matter(text)
            metadata = {
                "source": str(path.name),
                "path": str(path),
                "created": path.stat().st_ctime,
                "last_modified": path.stat().st_mtime,
                "last_accessed": path.stat().st_atime,
                **self._to_langchain_compatible_metadata(front_matter),
                **dataview_fields,
            }
            if tags or front_matter.get("tags"):
                metadata["tags"] = ",".join(
                    tags | set(front_matter.get("tags", []) or [])
                )
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
Source code for langchain_community.document_loaders.etherscan
import os
import re
from typing import Iterator, List
import requests
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseLoader
[docs]class EtherscanLoader(BaseLoader):
    """Load transactions from `Ethereum` mainnet.
    The Loader use Etherscan API to interact with Ethereum mainnet.
    ETHERSCAN_API_KEY environment variable must be set use this loader.
    """
[docs]    def __init__(
        self,
        account_address: str,
        api_key: str = "docs-demo",
        filter: str = "normal_transaction",
        page: int = 1,
        offset: int = 10,
        start_block: int = 0,
        end_block: int = 99999999,
        sort: str = "desc",
    ):
        self.account_address = account_address
        self.api_key = os.environ.get("ETHERSCAN_API_KEY") or api_key
        self.filter = filter
        self.page = page
        self.offset = offset
        self.start_block = start_block
        self.end_block = end_block
        self.sort = sort
        if not self.api_key:
            raise ValueError("Etherscan API key not provided")
        if not re.match(r"^0x[a-fA-F0-9]{40}$", self.account_address):
            raise ValueError(f"Invalid contract address {self.account_address}")
        if filter not in [
            "normal_transaction",
            "internal_transaction",
            "erc20_transaction",
            "eth_balance",
            "erc721_transaction",
            "erc1155_transaction",
        ]:
            raise ValueError(f"Invalid filter {filter}")