agent with memory unable to execute and throwing a output key error
Labels: name: bug

### System Info

python - 3.9
langchain - 0.0.213
OS - Mac Monterey

### Who can help?

_No response_

### Information

- [ ] The official example notebooks/scripts
- [X] My own modified scripts

### Related Components

- [ ] LLMs/Chat Models
- [ ] Embedding Models
- [ ] Prompts / Prompt Templates / Prompt Selectors
- [ ] Output Parsers
- [ ] Document Loaders
- [ ] Vector Stores / Retrievers
- [X] Memory
- [X] Agents / Agent Executors
- [ ] Tools / Toolkits
- [ ] Chains
- [ ] Callbacks/Tracing
- [ ] Async

### Reproduction


Code Snippet
```
from langchain import (
    LLMMathChain,
    OpenAI,
    SerpAPIWrapper,
    SQLDatabase,
    SQLDatabaseChain,
)
from langchain.agents import initialize_agent, Tool
from langchain.agents import AgentType
from langchain.chat_models import ChatOpenAI
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain.text_splitter import CharacterTextSplitter
from langchain.llms import OpenAI
from langchain.chains import RetrievalQA, ConversationChain
from langchain.agents import ZeroShotAgent, Tool, AgentExecutor
from langchain import OpenAI, LLMChain
from langchain.utilities import GoogleSearchAPIWrapper
from langchain import OpenAI, LLMMathChain, SerpAPIWrapper
from langchain.agents import initialize_agent, Tool
from langchain.chat_models import ChatOpenAI
from langchain.memory import ConversationBufferMemory, ConversationSummaryMemory, ConversationBufferWindowMemory
import time
from chainlit import AskUserMessage, Message, on_chat_start
import random
from langchain.document_loaders import TextLoader
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
from langchain.vectorstores import Milvus
from langchain.docstore.document import Document
from langchain.document_loaders.csv_loader import CSVLoader
from langchain.document_loaders import WebBaseLoader
from langchain.document_loaders import UnstructuredHTMLLoader
from langchain.chat_models import ChatOpenAI
from langchain.prompts import ChatPromptTemplate
from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate
from pathlib import Path
from langchain.document_loaders import WebBaseLoader
from langchain.prompts import PromptTemplate
import json
import os


llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo", openai_api_key=openai_api_key)

llm_math_chain = LLMMathChain.from_llm(llm=llm, verbose=True)

conversational_memory = ConversationBufferWindowMemory(
    memory_key='chat_history',
    k=50,
    return_messages=True,
    input_key="question", 
    output_key='output'
)

# load will vector db
relevant_parts = []
for p in Path(".").absolute().parts:
    relevant_parts.append(p)
    if relevant_parts[-3:] == ["langchain", "docs", "modules"]:
        break
        



def get_meta(data:str):
    split_data = [item.split(":") for item in data.split("\n")]

    # Creating a dictionary from the split data
    result = {}
    for item in split_data:
        if len(item) > 1:
            key = item[0].strip()
            value = item[1].strip()
            result[key] = value

    # Converting the dictionary to JSON format
    json_data = json.dumps(result)
    return json_data
    
    
template = """You're an customer care representative
 
You have the following products in your store based on the customer question. Answer politely to customer questions on any questions on product sold in the wesbite and provide product details. Send the product webpage links for each recommendation

{context}

"chat_history": {chat_history}

Question: {question}
Answer:"""
 

products_path = "ecommerce__20200101_20200131__10k_data_10.csv"


loader = CSVLoader(products_path, csv_args={
    'delimiter': ',',
    'quotechar': '"',
})

documents = loader.load()

text_splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=0, length_function = len)
sources = text_splitter.split_documents(documents)
source_chunks = []
splitter = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=100, length_function = len)

for source in sources:
    for chunk in splitter.split_text(source.page_content):
        chunk_metadata = json_meta = json.loads(get_meta(chunk))
        source_chunks.append(Document(page_content=chunk, metadata=chunk_metadata))

embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key, model="ada")

products_db = Chroma.from_documents(source_chunks, embeddings, collection_name="products")

prompt = PromptTemplate(template=template, input_variables=["context", "question", "chat_history"])

product_inquery_agent = RetrievalQA.from_chain_type(
    llm=llm,
    chain_type="stuff",
    retriever=products_db.as_retriever(search_kwargs={"k": 4}),
    chain_type_kwargs = {"prompt": prompt, "verbose": True, "memory": conversational_memory},
)


tools = [
    Tool(
        name="Product inquiries System",
        func=product_inquery_agent.run,
        description="useful for getting information about products, features, and specifications to make informed purchase decisions. Input should be a fully formed question.",
        return_direct=True,
    ),
]




prefix = """Have a conversation with a human as a customer representative agent, answering the following questions as best you can in a polite and friendly manner. You have access to the following tools:"""
suffix = """Begin!"

{chat_history}
Question: {input}
{agent_scratchpad}"""

prompt = ZeroShotAgent.create_prompt(
    tools,
    prefix=prefix,
    suffix=suffix,
    input_variables=["input", "chat_history", "agent_scratchpad"],
)
llm_chain = LLMChain(llm=OpenAI(temperature=0, openai_api_key=openai_api_key), prompt=prompt)

agent = ZeroShotAgent(llm_chain=llm_chain, tools=tools, verbose=True)
agent_chain = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, memory=conversational_memory)
```


### Expected behavior

query = "what products are available in your website ?"
response = agent_chain.run(query)
print(response)

The response should print the output captured but it throws a error with key not found

``` > Entering new  chain...
Thought: I need to find out what products are available
Action: Product inquiries System
Action Input: What products are available in your website?

> Entering new  chain...


> Entering new  chain...
Prompt after formatting:
You're an customer care representative
 
You have the following products in your store based on the customer question. Answer politely to customer questions on any questions on product sold in the wesbite and provide product details. Send the product webpage links for each recommendation

Uniq Id: cc2083338a16c3fe2f7895289d2e98fe
Product Name: ARTSCAPE Etched Glass 24" x 36" Window Film, 24-by-36-Inch
Brand Name: 
Asin: 
Category: Home & Kitchen | Home Décor | Window Treatments | Window Stickers & Films | Window Films
Upc Ean Code: 
List Price: 
Selling Price: $12.99
Quantity: 
Model Number: 01-0121
About Product: Make sure this fits by entering your model number. | The visual effect of textured glass and stained glass | Creates privacy / Provides UV protection | No adhesives / Applies easily | Patterns repeat to cover any size window | Made in USA
Product Specification: ProductDimensions:72x36x0inches|ItemWeight:11.2ounces|ShippingWeight:12.2ounces(Viewshippingratesandpolicies)|Manufacturer:Artscape|ASIN:B000Q3PRYA|DomesticShipping:ItemcanbeshippedwithinU.S.|InternationalShipping:ThisitemcanbeshippedtoselectcountriesoutsideoftheU.S.LearnMore|ShippingAdvisory:Thisitemmustbeshippedseparatelyfromotheritemsinyourorder.Additionalshippingchargeswillnotapply.|Itemmodelnumber:01-0121
Technical Details: show up to 2 reviews by default Product Description Artscape window films create the look of stained and etched glass. These thin, translucent films provide privacy while still allowing natural light to enter the room. Artscape films are easily applied to any smooth glass surface. They do not use adhesives and are easily removed if needed. They can be trimmed or combined to fit any size window. The images have a repeating pattern left to right and top to bottom and can be used either vertically or horizontally. These films provide UV protection and are the perfect decorative accent for windows that require continued privacy. Artscape patented products are all made in the USA. From the Manufacturer Artscape window films create the look of stained and etched glass. These thin, translucent films provide privacy while still allowing natural light to enter the room. Artscape films are easily applied to any smooth glass surface. They do not use adhesives and are easily removed if needed. They can be trimmed or combined to fit any size window. The images have a repeating pattern left to right and top to bottom and can be used either vertically or horizontally. These films provide UV protection and are the perfect decorative accent for windows that require continued privacy. Artscape patented products are all made in the USA. | 12.2 ounces (View shipping rates and policies)
Shipping Weight: 12.2 ounces
Product Dimensions: 
Image: [https://images-na.ssl-images-amazon.com/images/I/51iAe3LF7FL.jpg|https://images-na.ssl-images-amazon.com/images/I/417B20oEehL.jpg|https://images-na.ssl-images-amazon.com/images/I/41J3bDg633L.jpg|https://images-na.ssl-images-amazon.com/images/I/51hhnuGWwdL.jpg|https://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/transparent-pixel.jpg](https://images-na.ssl-images-amazon.com/images/I/51iAe3LF7FL.jpg%7Chttps://images-na.ssl-images-amazon.com/images/I/417B20oEehL.jpg%7Chttps://images-na.ssl-images-amazon.com/images/I/41J3bDg633L.jpg%7Chttps://images-na.ssl-images-amazon.com/images/I/51hhnuGWwdL.jpg%7Chttps://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/transparent-pixel.jpg)
Variants: 
Sku: 
Product Url: https://www.amazon.com/ARTSCAPE-Etched-Glass-Window-Film/dp/B000Q3PRYA
Stock: 
Product Details: 
Dimensions: 
Color: 
Ingredients:

Stock: 
Product Details: 
Dimensions: 
Color: 
Ingredients: 
Direction To Use: 
Is Amazon Seller: Y
Size Quantity Variant: 
Product Description:

Uniq Id: f8c32a45e507a177992973cf0d46d20c
Product Name: Terra by Battat – 4 Dinosaur Toys, Medium – Dinosaurs for Kids & Collectors, Scientifically Accurate & Designed by A Paleo-Artist; Age 3+ (4 Pc)
Brand Name: 
Asin: 
Category: 
Upc Ean Code: 
List Price: 
Selling Price: $18.66
Quantity: 
Model Number: AN4054Z
About Product: Make sure this fits by entering your model number. | 4 medium-sized dinosaurs for kids, with lifelike pose, accurate ratio, and exquisitely detailed paint | Includes: Parasaurolophus walkeri, Stegosaurus ungulatus, Pachyrhinosaurus, and euoplocephalus tutus toy dinos | Museum Quality: classic toy dinosaurs designed by an internationally renowned paleo-artist | Educational toy: dinosaur toys for kids spark curiosity about paleontology, science, and natural History | Dimensions: Dimensions: miniature figurines measure 6.25-8.5 (L) 1.5-2.25 (W) 2.25-3.5 (H) inches approximately toys 6.25-8.5 (L) 1.5-2.25 (W) 2.25-3.5 (H) inches approximately | No batteries required, just imagination! | Earth-friendly recyclable packaging | Age: suggested for ages 3+ | Collect them all! Discover the entire Terra by Battat family of animal toy figurines and dinosaur playsets!
Product Specification: ProductDimensions:8.7x3.9x3.4inches|ItemWeight:15.8ounces|ShippingWeight:1.4pounds(Viewshippingratesandpolicies)|ASIN:B07PF1R8LS|Itemmodelnumber:AN4054Z|Manufacturerrecommendedage:36months-10years
Technical Details: Go to your orders and start the return Select the ship method Ship it! | Go to your orders and start the return Select the ship method Ship it! | show up to 2 reviews by default Look out! Here come 4 incredible dinosaur toys that look like the real thing! Sought after by both kids and collectors, These scientifically precise and very collectable dinosaur replicas also feature bright, beautiful colors and look amazing in any dinosaur playset. And, they're made with high quality material, so these dinos will never go extinct! The intimidating-looking, and instantly recognizable Stegosaurus was the largest of plate-backed prehistoric herbivores. Scientists have suggested that the Stegosaurus’ iconic plates were for defense against predators, but this is unlikely as the plates were quite thin. Others have said the Stegosaurus’ spiked tail may have been used as a weapon. Terra’s Stegosaurus toy accurately recreates the formidable size and terrifying presence of this iconic predator. In particular, the large, heavily built frame and rounded backs of the mighty Stegosaurus are faithfully depicted in the dinosaur replica. This highly collectable playset also includes a toy Parasaurolophus, Stegosaurus, Pachyrhinosaurus, and euoplocephalus tutus. Collect your own miniature versions of 4 powerful and iconic dinosaurs from Terra by Battat! | 1.4 pounds (View shipping rates and policies)
Shipping Weight: 1.4 pounds
Product Dimensions:

Uniq Id: e04b990e95bf73bbe6a3fa09785d7cd0
Product Name: Woodstock- Collage 500 pc Puzzle
Brand Name: 
Asin: 
Category: Toys & Games | Puzzles | Jigsaw Puzzles
Upc Ean Code: 
List Price: 
Selling Price: $17.49
Quantity: 
Model Number: 62151
About Product: Make sure this fits by entering your model number. | Puzzle has 500 pieces | Completed puzzle measure 14 x 19 | 100% officially licensed merchandise | Great for fans & puzzlers alike
Product Specification: ProductDimensions:1.9x8x10inches|ItemWeight:13.4ounces|ShippingWeight:13.4ounces(Viewshippingratesandpolicies)|ASIN:B07MX21WWX|Itemmodelnumber:62151|Manufacturerrecommendedage:14yearsandup
Technical Details: show up to 2 reviews by default 100% Officially licensed merchandise; complete puzzle measures 14 x 19 in. | 13.4 ounces (View shipping rates and policies)
Shipping Weight: 13.4 ounces
Product Dimensions: 
Image: [https://images-na.ssl-images-amazon.com/images/I/61plo8Xv4vL.jpg|https://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/transparent-pixel.jpg](https://images-na.ssl-images-amazon.com/images/I/61plo8Xv4vL.jpg%7Chttps://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/transparent-pixel.jpg)
Variants: 
Sku: 
Product Url: https://www.amazon.com/Woodstock-Collage-500-pc-Puzzle/dp/B07MX21WWX
Stock: 
Product Details: 
Dimensions: 
Color: 
Ingredients: 
Direction To Use: 
Is Amazon Seller: Y
Size Quantity Variant: 
Product Description:

"chat_history": []

Question: What products are available in your website?
Answer:

> Finished chain.

> Finished chain.
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[42], line 2
      1 query = "what products are available in your website ?"
----> 2 response = agent_chain.run(query)
      3 print(response)

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:290, in Chain.run(self, callbacks, tags, *args, **kwargs)
    288     if len(args) != 1:
    289         raise ValueError("`run` supports only one positional argument.")
--> 290     return self(args[0], callbacks=callbacks, tags=tags)[_output_key]
    292 if kwargs and not args:
    293     return self(kwargs, callbacks=callbacks, tags=tags)[_output_key]

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:166, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, include_run_info)
    164 except (KeyboardInterrupt, Exception) as e:
    165     run_manager.on_chain_error(e)
--> 166     raise e
    167 run_manager.on_chain_end(outputs)
    168 final_outputs: Dict[str, Any] = self.prep_outputs(
    169     inputs, outputs, return_only_outputs
    170 )

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:160, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, include_run_info)
    154 run_manager = callback_manager.on_chain_start(
    155     dumpd(self),
    156     inputs,
    157 )
    158 try:
    159     outputs = (
--> 160         self._call(inputs, run_manager=run_manager)
    161         if new_arg_supported
    162         else self._call(inputs)
    163     )
    164 except (KeyboardInterrupt, Exception) as e:
    165     run_manager.on_chain_error(e)

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/agents/agent.py:987, in AgentExecutor._call(self, inputs, run_manager)
    985 # We now enter the agent loop (until it returns something).
    986 while self._should_continue(iterations, time_elapsed):
--> 987     next_step_output = self._take_next_step(
    988         name_to_tool_map,
    989         color_mapping,
    990         inputs,
    991         intermediate_steps,
    992         run_manager=run_manager,
    993     )
    994     if isinstance(next_step_output, AgentFinish):
    995         return self._return(
    996             next_step_output, intermediate_steps, run_manager=run_manager
    997         )

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/agents/agent.py:850, in AgentExecutor._take_next_step(self, name_to_tool_map, color_mapping, inputs, intermediate_steps, run_manager)
    848         tool_run_kwargs["llm_prefix"] = ""
    849     # We then call the tool on the tool input to get an observation
--> 850     observation = tool.run(
    851         agent_action.tool_input,
    852         verbose=self.verbose,
    853         color=color,
    854         callbacks=run_manager.get_child() if run_manager else None,
    855         **tool_run_kwargs,
    856     )
    857 else:
    858     tool_run_kwargs = self.agent.tool_run_logging_kwargs()

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/tools/base.py:299, in BaseTool.run(self, tool_input, verbose, start_color, color, callbacks, **kwargs)
    297 except (Exception, KeyboardInterrupt) as e:
    298     run_manager.on_tool_error(e)
--> 299     raise e
    300 else:
    301     run_manager.on_tool_end(
    302         str(observation), color=color, name=self.name, **kwargs
    303     )

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/tools/base.py:271, in BaseTool.run(self, tool_input, verbose, start_color, color, callbacks, **kwargs)
    268 try:
    269     tool_args, tool_kwargs = self._to_args_and_kwargs(parsed_input)
    270     observation = (
--> 271         self._run(*tool_args, run_manager=run_manager, **tool_kwargs)
    272         if new_arg_supported
    273         else self._run(*tool_args, **tool_kwargs)
    274     )
    275 except ToolException as e:
    276     if not self.handle_tool_error:

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/tools/base.py:414, in Tool._run(self, run_manager, *args, **kwargs)
    411 """Use the tool."""
    412 new_argument_supported = signature(self.func).parameters.get("callbacks")
    413 return (
--> 414     self.func(
    415         *args,
    416         callbacks=run_manager.get_child() if run_manager else None,
    417         **kwargs,
    418     )
    419     if new_argument_supported
    420     else self.func(*args, **kwargs)
    421 )

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:290, in Chain.run(self, callbacks, tags, *args, **kwargs)
    288     if len(args) != 1:
    289         raise ValueError("`run` supports only one positional argument.")
--> 290     return self(args[0], callbacks=callbacks, tags=tags)[_output_key]
    292 if kwargs and not args:
    293     return self(kwargs, callbacks=callbacks, tags=tags)[_output_key]

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:166, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, include_run_info)
    164 except (KeyboardInterrupt, Exception) as e:
    165     run_manager.on_chain_error(e)
--> 166     raise e
    167 run_manager.on_chain_end(outputs)
    168 final_outputs: Dict[str, Any] = self.prep_outputs(
    169     inputs, outputs, return_only_outputs
    170 )

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:160, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, include_run_info)
    154 run_manager = callback_manager.on_chain_start(
    155     dumpd(self),
    156     inputs,
    157 )
    158 try:
    159     outputs = (
--> 160         self._call(inputs, run_manager=run_manager)
    161         if new_arg_supported
    162         else self._call(inputs)
    163     )
    164 except (KeyboardInterrupt, Exception) as e:
    165     run_manager.on_chain_error(e)

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/retrieval_qa/base.py:120, in BaseRetrievalQA._call(self, inputs, run_manager)
    117 question = inputs[self.input_key]
    119 docs = self._get_docs(question)
--> 120 answer = self.combine_documents_chain.run(
    121     input_documents=docs, question=question, callbacks=_run_manager.get_child()
    122 )
    124 if self.return_source_documents:
    125     return {self.output_key: answer, "source_documents": docs}

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:293, in Chain.run(self, callbacks, tags, *args, **kwargs)
    290     return self(args[0], callbacks=callbacks, tags=tags)[_output_key]
    292 if kwargs and not args:
--> 293     return self(kwargs, callbacks=callbacks, tags=tags)[_output_key]
    295 if not kwargs and not args:
    296     raise ValueError(
    297         "`run` supported with either positional arguments or keyword arguments,"
    298         " but none were provided."
    299     )

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:168, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, include_run_info)
    166     raise e
    167 run_manager.on_chain_end(outputs)
--> 168 final_outputs: Dict[str, Any] = self.prep_outputs(
    169     inputs, outputs, return_only_outputs
    170 )
    171 if include_run_info:
    172     final_outputs[RUN_KEY] = RunInfo(run_id=run_manager.run_id)

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/chains/base.py:233, in Chain.prep_outputs(self, inputs, outputs, return_only_outputs)
    231 self._validate_outputs(outputs)
    232 if self.memory is not None:
--> 233     self.memory.save_context(inputs, outputs)
    234 if return_only_outputs:
    235     return outputs

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/memory/chat_memory.py:34, in BaseChatMemory.save_context(self, inputs, outputs)
     32 def save_context(self, inputs: Dict[str, Any], outputs: Dict[str, str]) -> None:
     33     """Save context from this conversation to buffer."""
---> 34     input_str, output_str = self._get_input_output(inputs, outputs)
     35     self.chat_memory.add_user_message(input_str)
     36     self.chat_memory.add_ai_message(output_str)

File /opt/miniconda3/envs/discord_dev/lib/python3.9/site-packages/langchain/memory/chat_memory.py:30, in BaseChatMemory._get_input_output(self, inputs, outputs)
     28 else:
     29     output_key = self.output_key
---> 30 return inputs[prompt_input_key], outputs[output_key]

KeyError: 'output' ```