aignosi/langchaing-docs-chatgpt-plugin huggingface dataset:
Source code for langchain.document_loaders.roam
"""Loader that loads Roam directory dump."""
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class RoamLoader(BaseLoader):
    """Loader that loads Roam files from disk."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        ps = list(Path(self.file_path).glob("**/*.md"))
        docs = []
        for p in ps:
            with open(p) as f:
                text = f.read()
            metadata = {"source": str(p)}
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
values,
            "openai_organization",
            "OPENAI_ORGANIZATION",
            default="",
        )
        try:
            import openai
            openai.api_key = openai_api_key
            if openai_organization:
                openai.organization = openai_organization
            values["client"] = openai.Moderation
        except ImportError:
            raise ValueError(
                "Could not import openai python package. "
                "Please install it with `pip install openai`."
            )
        return values
    @property
    def input_keys(self) -> List[str]:
        """Expect input key.
        :meta private:
        """
        return [self.input_key]
    @property
    def output_keys(self) -> List[str]:
        """Return output key.
        :meta private:
        """
        return [self.output_key]
    def _moderate(self, text: str, results: dict) -> str:
        if results["flagged"]:
            error_str = "Text was found that violates OpenAI's content policy."
            if self.error:
                raise ValueError(error_str)
            else:
                return error_str
        return text
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        text = inputs[self.input_key]
        results = self.client.create(text)
        output = self._moderate(text, results["results"][0])
        return {self.output_key: output}
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
reduce_chain = StuffDocumentsChain(llm_chain=llm_chain, callbacks=callbacks)
        combine_documents_chain = MapReduceDocumentsChain(
            llm_chain=llm_chain,
            combine_document_chain=reduce_chain,
            callbacks=callbacks,
        )
        return cls(
            combine_documents_chain=combine_documents_chain,
            text_splitter=text_splitter,
            callbacks=callbacks,
            **kwargs,
        )
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @property
    def input_keys(self) -> List[str]:
        """Expect input key.
        :meta private:
        """
        return [self.input_key]
    @property
    def output_keys(self) -> List[str]:
        """Return output key.
        :meta private:
        """
        return [self.output_key]
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        # Split the larger text into smaller chunks.
        texts = self.text_splitter.split_text(inputs[self.input_key])
        docs = [Document(page_content=text) for text in texts]
        outputs = self.combine_documents_chain.run(
            input_documents=docs, callbacks=_run_manager.get_child()
        )
        return {self.output_key: outputs}
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
:meta private:
        """
        return [self.output_key]
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        try:
            from bs4 import BeautifulSoup  # noqa: F401
        except ImportError:
            raise ValueError(
                "Could not import bs4 python package. "
                "Please install it with `pip install bs4`."
            )
        return values
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        from bs4 import BeautifulSoup
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        # Other keys are assumed to be needed for LLM prediction
        other_keys = {k: v for k, v in inputs.items() if k != self.input_key}
        url = inputs[self.input_key]
        res = self.requests_wrapper.get(url)
        # extract the text from the html
        soup = BeautifulSoup(res, "html.parser")
        other_keys[self.requests_key] = soup.get_text()[: self.text_length]
        result = self.llm_chain.predict(
            callbacks=_run_manager.get_child(), **other_keys
        )
        return {self.output_key: result}
    @property
    def _chain_type(self) -> str:
        return "llm_requests_chain"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
return [
                self.prompt.output_parser.parse(res[self.output_key]) for res in result
            ]
        else:
            return result
[docs]    async def aapply_and_parse(
        self, input_list: List[Dict[str, Any]], callbacks: Callbacks = None
    ) -> Sequence[Union[str, List[str], Dict[str, str]]]:
        """Call apply and then parse the results."""
        result = await self.aapply(input_list, callbacks=callbacks)
        return self._parse_result(result)
    @property
    def _chain_type(self) -> str:
        return "llm_chain"
[docs]    @classmethod
    def from_string(cls, llm: BaseLanguageModel, template: str) -> Chain:
        """Create LLMChain from LLM and template."""
        prompt_template = PromptTemplate.from_template(template)
        return cls(llm=llm, prompt=prompt_template)
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
_run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        _input = inputs[self.input_key]
        color_mapping = get_color_mapping([str(i) for i in range(len(self.chains))])
        for i, chain in enumerate(self.chains):
            _input = await chain.arun(_input, callbacks=callbacks)
            if self.strip_outputs:
                _input = _input.strip()
            await _run_manager.on_text(
                _input, color=color_mapping[str(i)], end="\n", verbose=self.verbose
            )
        return {self.output_key: _input}
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
Source code for langchain.chains.transform
"""Chain that runs an arbitrary python function."""
from typing import Callable, Dict, List, Optional
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
[docs]class TransformChain(Chain):
    """Chain transform chain output.
    Example:
        .. code-block:: python
            from langchain import TransformChain
            transform_chain = TransformChain(input_variables=["text"],
             output_variables["entities"], transform=func())
    """
    input_variables: List[str]
    output_variables: List[str]
    transform: Callable[[Dict[str, str]], Dict[str, str]]
    @property
    def input_keys(self) -> List[str]:
        """Expect input keys.
        :meta private:
        """
        return self.input_variables
    @property
    def output_keys(self) -> List[str]:
        """Return output keys.
        :meta private:
        """
        return self.output_variables
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        return self.transform(inputs)
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
raise ValueError(f"search_type of {self.search_type} not allowed.")
        return docs
    async def _aget_docs(self, question: str) -> List[Document]:
        raise NotImplementedError("VectorDBQA does not support async")
    @property
    def _chain_type(self) -> str:
        """Return the chain type."""
        return "vector_db_qa"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
combine_docs_chain_kwargs = combine_docs_chain_kwargs or {}
        doc_chain = load_qa_chain(
            llm,
            chain_type=chain_type,
            **combine_docs_chain_kwargs,
        )
        condense_question_chain = LLMChain(llm=llm, prompt=condense_question_prompt)
        return cls(
            vectorstore=vectorstore,
            combine_docs_chain=doc_chain,
            question_generator=condense_question_chain,
            **kwargs,
        )
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
num_docs -= 1
                token_count -= tokens[num_docs]
        return docs[:num_docs]
    def _get_docs(self, inputs: Dict[str, Any]) -> List[Document]:
        question = inputs[self.question_key]
        docs = self.vectorstore.similarity_search(
            question, k=self.k, **self.search_kwargs
        )
        return self._reduce_tokens_below_limit(docs)
    async def _aget_docs(self, inputs: Dict[str, Any]) -> List[Document]:
        raise NotImplementedError("VectorDBQAWithSourcesChain does not support async")
    @root_validator()
    def raise_deprecation(cls, values: Dict) -> Dict:
        warnings.warn(
            "`VectorDBQAWithSourcesChain` is deprecated - "
            "please use `from langchain.chains import RetrievalQAWithSourcesChain`"
        )
        return values
    @property
    def _chain_type(self) -> str:
        return "vector_db_qa_with_sources_chain"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
docs = self.retriever.get_relevant_documents(question)
        return self._reduce_tokens_below_limit(docs)
    async def _aget_docs(self, inputs: Dict[str, Any]) -> List[Document]:
        question = inputs[self.question_key]
        docs = await self.retriever.aget_relevant_documents(question)
        return self._reduce_tokens_below_limit(docs)
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, List]:
        docs = self.text_splitter.create_documents([inputs[self.input_key]])
        results = self.llm_chain.generate(
            [{"text": d.page_content} for d in docs], run_manager=run_manager
        )
        qa = [json.loads(res[0].text) for res in results.generations]
        return {self.output_key: qa}
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: BasePromptTemplate = PROMPT,
        **kwargs: Any,
    ) -> LLMMathChain:
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(llm_chain=llm_chain, **kwargs)
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
)
        _run_manager.on_text(t, color="green", verbose=self.verbose)
        t = t.strip()
        try:
            parser = self.llm_chain.prompt.output_parser
            command_list = parser.parse(t)  # type: ignore[union-attr]
        except OutputParserException as e:
            _run_manager.on_chain_error(e, verbose=self.verbose)
            raise e
        if self.verbose:
            _run_manager.on_text("\nCode: ", verbose=self.verbose)
            _run_manager.on_text(
                str(command_list), color="yellow", verbose=self.verbose
            )
        output = self.bash_process.run(command_list)
        _run_manager.on_text("\nAnswer: ", verbose=self.verbose)
        _run_manager.on_text(output, color="yellow", verbose=self.verbose)
        return {self.output_key: output}
    @property
    def _chain_type(self) -> str:
        return "llm_bash_chain"
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: BasePromptTemplate = PROMPT,
        **kwargs: Any,
    ) -> LLMBashChain:
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(llm_chain=llm_chain, **kwargs)
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
return list(np.array(embeddings).mean(axis=0))
[docs]    def embed_query(self, text: str) -> List[float]:
        """Generate a hypothetical document and embedded it."""
        var_name = self.llm_chain.input_keys[0]
        result = self.llm_chain.generate([{var_name: text}])
        documents = [generation.text for generation in result.generations[0]]
        embeddings = self.embed_documents(documents)
        return self.combine_embeddings(embeddings)
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        """Call the internal llm chain."""
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        return self.llm_chain(inputs, callbacks=_run_manager.get_child())
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        base_embeddings: Embeddings,
        prompt_key: str,
        **kwargs: Any,
    ) -> HypotheticalDocumentEmbedder:
        """Load and use LLMChain for a specific prompt key."""
        prompt = PROMPT_MAP[prompt_key]
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(base_embeddings=base_embeddings, llm_chain=llm_chain, **kwargs)
    @property
    def _chain_type(self) -> str:
        return "hyde_chain"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
requests_wrapper = TextRequestsWrapper(headers=headers)
        get_answer_chain = LLMChain(llm=llm, prompt=api_response_prompt)
        return cls(
            api_request_chain=get_request_chain,
            api_answer_chain=get_answer_chain,
            requests_wrapper=requests_wrapper,
            api_docs=api_docs,
            **kwargs,
        )
    @property
    def _chain_type(self) -> str:
        return "api_chain"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
) -> GraphQAChain:
        """Initialize from LLM."""
        qa_chain = LLMChain(llm=llm, prompt=qa_prompt)
        entity_chain = LLMChain(llm=llm, prompt=entity_prompt)
        return cls(
            qa_chain=qa_chain,
            entity_extraction_chain=entity_chain,
            **kwargs,
        )
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        """Extract entities, look up info and answer question."""
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        question = inputs[self.input_key]
        entity_string = self.entity_extraction_chain.run(question)
        _run_manager.on_text("Entities Extracted:", end="\n", verbose=self.verbose)
        _run_manager.on_text(
            entity_string, color="green", end="\n", verbose=self.verbose
        )
        entities = get_entities(entity_string)
        context = ""
        for entity in entities:
            triplets = self.graph.get_entity_knowledge(entity)
            context += "\n".join(triplets)
        _run_manager.on_text("Full Context:", end="\n", verbose=self.verbose)
        _run_manager.on_text(context, color="green", end="\n", verbose=self.verbose)
        result = self.qa_chain(
            {"question": question, "context": context},
            callbacks=_run_manager.get_child(),
        )
        return {self.output_key: result[self.qa_chain.output_key]}
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
question = inputs[self.input_key]
        output = self.question_to_checked_assertions_chain(
            {"question": question}, callbacks=_run_manager.get_child()
        )
        return {self.output_key: output["revised_statement"]}
    @property
    def _chain_type(self) -> str:
        return "llm_checker_chain"
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        create_draft_answer_prompt: PromptTemplate = CREATE_DRAFT_ANSWER_PROMPT,
        list_assertions_prompt: PromptTemplate = LIST_ASSERTIONS_PROMPT,
        check_assertions_prompt: PromptTemplate = CHECK_ASSERTIONS_PROMPT,
        revised_answer_prompt: PromptTemplate = REVISED_ANSWER_PROMPT,
        **kwargs: Any,
    ) -> LLMCheckerChain:
        question_to_checked_assertions_chain = (
            _load_question_to_checked_assertions_chain(
                llm,
                create_draft_answer_prompt,
                list_assertions_prompt,
                check_assertions_prompt,
                revised_answer_prompt,
            )
        )
        return cls(
            question_to_checked_assertions_chain=question_to_checked_assertions_chain,
            **kwargs,
        )
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
# Other keys are assumed to be needed for LLM prediction
        other_keys: Dict = {k: v for k, v in inputs.items() if k != self.input_key}
        other_keys[self.combine_docs_chain.input_key] = docs
        return self.combine_docs_chain(
            other_keys, return_only_outputs=True, callbacks=_run_manager.get_child()
        )
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
critiques_and_revisions.append((critique, revision))
            _run_manager.on_text(
                text=f"Applying {constitutional_principle.name}..." + "\n\n",
                verbose=self.verbose,
                color="green",
            )
            _run_manager.on_text(
                text="Critique: " + critique + "\n\n",
                verbose=self.verbose,
                color="blue",
            )
            _run_manager.on_text(
                text="Updated response: " + revision + "\n\n",
                verbose=self.verbose,
                color="yellow",
            )
        final_output: Dict[str, Any] = {"output": response}
        if self.return_intermediate_steps:
            final_output["initial_output"] = initial_response
            final_output["critiques_and_revisions"] = critiques_and_revisions
        return final_output
    @staticmethod
    def _parse_critique(output_string: str) -> str:
        if "Revision request:" not in output_string:
            return output_string
        output_string = output_string.split("Revision request:")[0]
        if "\n\n" in output_string:
            output_string = output_string.split("\n\n")[0]
        return output_string
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
f"The input key {input_key} was also found in the memory keys "
                f"({memory_keys}) - please provide keys that don't overlap."
            )
        prompt_variables = values["prompt"].input_variables
        expected_keys = memory_keys + [input_key]
        if set(expected_keys) != set(prompt_variables):
            raise ValueError(
                "Got unexpected prompt input variables. The prompt expects "
                f"{prompt_variables}, but got {memory_keys} as inputs from "
                f"memory, and {input_key} as the normal input key."
            )
        return values
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
)
            initial_response = response.strip() + " " + "".join(tokens)
            if not low_confidence_spans:
                response = initial_response
                final_response, finished = self.output_parser.parse(response)
                if finished:
                    return {self.output_keys[0]: final_response}
                continue
            marginal, finished = self._do_retrieval(
                low_confidence_spans,
                _run_manager,
                user_input,
                response,
                initial_response,
            )
            response = response.strip() + " " + marginal
            if finished:
                break
        return {self.output_keys[0]: response}
[docs]    @classmethod
    def from_llm(
        cls, llm: BaseLanguageModel, max_generation_len: int = 32, **kwargs: Any
    ) -> FlareChain:
        question_gen_chain = QuestionGeneratorChain(llm=llm)
        response_llm = OpenAI(
            max_tokens=max_generation_len, model_kwargs={"logprobs": 1}, temperature=0
        )
        response_chain = _OpenAIResponseChain(llm=response_llm)
        return cls(
            question_generator_chain=question_gen_chain,
            response_chain=response_chain,
            **kwargs,
        )
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
output["intermediate_steps"] = code
        return output
[docs]    @classmethod
    def from_math_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PALChain:
        """Load PAL from math prompt."""
        llm_chain = LLMChain(llm=llm, prompt=MATH_PROMPT)
        return cls(
            llm_chain=llm_chain,
            stop="\n\n",
            get_answer_expr="print(solution())",
            **kwargs,
        )
[docs]    @classmethod
    def from_colored_object_prompt(
        cls, llm: BaseLanguageModel, **kwargs: Any
    ) -> PALChain:
        """Load PAL from colored object prompt."""
        llm_chain = LLMChain(llm=llm, prompt=COLORED_OBJECT_PROMPT)
        return cls(
            llm_chain=llm_chain,
            stop="\n\n\n",
            get_answer_expr="print(answer)",
            **kwargs,
        )
    @property
    def _chain_type(self) -> str:
        return "pal_chain"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.
run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _table_names = self.sql_chain.database.get_usable_table_names()
        table_names = ", ".join(_table_names)
        llm_inputs = {
            "query": inputs[self.input_key],
            "table_names": table_names,
        }
        _lowercased_table_names = [name.lower() for name in _table_names]
        table_names_from_chain = self.decider_chain.predict_and_parse(**llm_inputs)
        table_names_to_use = [
            name
            for name in table_names_from_chain
            if name.lower() in _lowercased_table_names
        ]
        _run_manager.on_text("Table names to use:", end="\n", verbose=self.verbose)
        _run_manager.on_text(
            str(table_names_to_use), color="yellow", verbose=self.verbose
        )
        new_inputs = {
            self.sql_chain.input_key: inputs[self.input_key],
            "table_names_to_use": table_names_to_use,
        }
        return self.sql_chain(
            new_inputs, callbacks=_run_manager.get_child(), return_only_outputs=True
        )
    @property
    def _chain_type(self) -> str:
        return "sql_database_sequential_chain"
By Harrison Chase
    
      © Copyright 2023, Harrison Chase.
      
  Last updated on May 18, 2023.