Issue: Getting CLOSE_WAIT and too many files open error using ChatAnthropic and StuffDocumentsChain
Labels: name: auto:bug, name: auto:question, name: area: models

### Issue you'd like to raise.

Hi, I am trying claude-2 model using the ChatAnthropic library and iterating over my data to call the model end for predictions. Since it's a chain of input, I am using StuffDocumentsChain. I am facing two issues currently.
1. CLOSE_WAIT error after the connection is established with cloud front with anthropic, it keeps creating a new connection for each iteration of data and goes into CLOSE_WAIT state after the process is completed for that iteration after some time.
2. When too many connections go into the close wait state the application gives too many files open error due to the file descriptor handling too many connections.

## Solutions Tried

1. Giving request_default_timeout to handle the CLOSE_WAIT error. 
2. Tried creating a function try/catch and final to close any objects within the block.

## Code

```
import warnings
warnings.filterwarnings("ignore")
import jaconv
import numpy as np
import pandas as pd
from langchain import PromptTemplate
from fuzzysearch import find_near_matches
from langchain.vectorstores import Chroma
from langchain.chat_models import ChatAnthropic
from langchain.document_loaders import PDFPlumberLoader
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.schema import HumanMessage, SystemMessage
from langchain.schema.output_parser import OutputParserException
from langchain.chains import LLMChain, StuffDocumentsChain
from langchain.retrievers import BM25Retriever, EnsembleRetriever
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.output_parsers import StructuredOutputParser, ResponseSchema
from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
import time
from json.decoder import JSONDecodeError
from loguru import logger

output_parser = StructuredOutputParser.from_response_schemas([
    ResponseSchema(
        name="answer",
        description="answer to the user's question",
        type="string"
    ),
])
format_instructions = output_parser.get_format_instructions()


def load_data_(file_path):
    return PDFPlumberLoader(file_path).load_and_split()


def retriever_(docs):
    return EnsembleRetriever(
        retrievers=[
            BM25Retriever.from_documents(docs, k=4),
            Chroma.from_documents(
                docs,
                OpenAIEmbeddings(
                    chunk_size=200,
                    max_retries=60,
                    show_progress_bar=True
                )
            ).as_retriever(search_kwargs={"k": 4})
        ],
        weights=[0.5, 0.5]
    )

def qa_chain():
    start_qa = time.time()
    qa_chain_output = StuffDocumentsChain(
        llm_chain=LLMChain(
            llm=ChatAnthropic(
                model_name="claude-2",
                temperature=0,
                top_p=1,
                max_tokens_to_sample=500000,
                default_request_timeout=30
            ),
            prompt=ChatPromptTemplate(
                messages=[
                    SystemMessage(content='You are a world class & knowledgeable product catalog assistance.'),
                    HumanMessagePromptTemplate.from_template('Context:\n{context}'),
                    HumanMessagePromptTemplate.from_template('{format_instructions}'),
                    HumanMessagePromptTemplate.from_template('Questions:\n{question}'),
                    HumanMessage(content='Tips: Make sure to answer in the correct format.'),
                ],
                partial_variables={"format_instructions": format_instructions}
            )
        ),
        document_variable_name='context',
        document_prompt=PromptTemplate(
            template='Content: {page_content}',
            input_variables=['page_content'],
        )
    )
    logger.info("Finished QA Chain in {}",time.time()-start_qa)
    return qa_chain_output

def generate(input_data, retriever):
    start_generate = time.time()
    doc_query = jaconv.z2h(
        jaconv.normalize(
            "コード: {}\n製品の種類: {}\nメーカー品名: {}".format(
                str(input_data["Product Code"]),
                str(input_data["Product Type"]),
                str(input_data["Product Name"])
            ),
            "NFKC"
        ),
        kana=False,
        digit=True,
        ascii=True
    )
    docs = RecursiveCharacterTextSplitter(
        chunk_size=512,
        chunk_overlap=128
    ).split_documents(
        retriever.get_relevant_documents(doc_query)
    )
    docs = [
        doc
        for doc in docs
        if len(
            find_near_matches(
                str(input_data["Product Code"]),
                str(doc.page_content),
                max_l_dist=1
            )
        ) > 0
    ]
    pages = list(set([str(doc.metadata["page"]) for doc in docs]))

    question = (
        "Analyze the provided context and understand to extract the following attributes as precise as possible for {}"
        "Attributes:\n"
        "{}\n"
        "Return it in this JSON format: {}. "
        "if no information found or not sure return \"None\"."
    )
    generate_output = output_parser.parse(
        qa_chain().run({
            "input_documents": docs,
            "question": question.format(
                str(input_data["Product Code"]),
                '\n'.join([
                    f" {i + 1}. What is the value of \"{attrib}\"?"
                    for i, attrib in enumerate(input_data["Attributes"])
                ]),
                str({"answer": {attrib: f"value of {attrib}" for attrib in input_data["Attributes"]}}),
            )
        })
    )["answer"], ';'.join(pages)
    logger.info("Finished QA Chain in {}",time.time()-start_generate)
    return generate_output


def predict(pdf_file_path:str, csv_file_path:str,output_csv_file_path:str):
    # load PDF data
    start_load = time.time()
    logger.info("Started loading the pdf")
    documents = load_data_(pdf_file_path)
    logger.info("Finished loading the pdf in {}",time.time()-start_load)
    try:
        start_retriever = time.time()
        logger.info("Started retriever the pdf")
        retriever = retriever_(documents)
        logger.info("Finished retriever the pdf in {}",time.time()-start_retriever)
    except Exception as e:
        logger.error(f"Error in Retriever: {e}")
    # Load CSV
    start_load_csv = time.time()
    logger.info("Started Reading Csv")
    df = pd.read_csv(
        csv_file_path,
        low_memory=False,
        dtype=object
    ).replace(np.nan, 'None')
    logger.info("Finished Reading Csv in {}",time.time()-start_load_csv)
    # Inference
    index = 0
    result_df = list()
    start_generate = time.time()
    logger.info("Started Generate Function")
    for code, dfx in df.groupby('Product Code'):
        start_generate_itr = time.time()
        try:
            logger.info("Reached index {}",index)
            logger.info("Reached index code {}",code)
            index = index + 1
            result, page = generate(
                {
                    'Product Code': code,
                    'Product Type': dfx['Product Type'].tolist()[0],
                    'Product Name': dfx['Product Name'].tolist()[0],
                    'Attributes': dfx['Attributes'].tolist()
                },
                retriever
            )
            dfx['Value'] = dfx['Attributes'].apply(lambda attrib: result.get(attrib, 'None'))
            dfx['Page'] = page
            logger.info("Generate Function 1 iterations {}",time.time()-start_generate_itr)
        except OutputParserException as e:
            dfx['Value'] = "None"
            dfx['Page'] = "None"
            logger.info("Generate Function 1 iterations {}",time.time()-start_generate_itr)
            logger.info("JSONDecodeError Exception Occurred in {}",e)
        result_df.append(dfx)
    logger.info("Finished Generate Function in {}",time.time()-start_generate)
    try:
        result_df = pd.concat(result_df)
        result_df.to_csv(output_csv_file_path)
    except FileNotFoundError as e:
        logger.error("FileNotFoundError Exception Occurred in {}",e)
    return result_df

# df = predict(f"{PDF_FILE}", f"{CSV_FILE}","output.csv")
```



### Suggestion:

I would like to know how to handle the connection CLOSE_WAIT error since I am handling a big amount of data to be processed through Anthropic claude-2