langserve with Llamacpp
Labels: name: auto:bug, name: auto:question, name: area: models, name: integration: chroma

I am trying to use langserve with langchain llamacpp like this (chain.py):

```from flask import Flask, jsonify, request
from langchain.chains import RetrievalQA,ConversationalRetrievalChain
from langchain.embeddings import HuggingFaceInstructEmbeddings
from langchain.llms import LlamaCpp
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler


from prompt_template_utils import get_prompt_template


from langchain.vectorstores import Chroma, FAISS
from werkzeug.utils import secure_filename

from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME

DEVICE_TYPE = "cuda" if torch.cuda.is_available() else "cpu"
SHOW_SOURCES = True
logging.info(f"Running on: {DEVICE_TYPE}")
logging.info(f"Display Source Documents set to: {SHOW_SOURCES}")

MODEL_PATH = '../llama-2-7b-32k-instruct.Q6_K.gguf'

def load_model():

    callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])

    n_gpu_layers = 30  # Change this value based on your model and your GPU VRAM pool.
    n_batch = 512  # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.

    llm = LlamaCpp(
            model_path=MODEL_PATH,
            n_ctx = 8192,
            n_gpu_layers=n_gpu_layers,
            n_batch=n_batch,
            repeat_penalty=1,
            temperature=0.2,
            max_tokens=100,
            top_p=0.9,
            top_k = 50,
            rope_freq_scale=0.125,
            stop = ["
"],
            # callback_manager=callback_manager, 
            streaming=True,
            verbose=True, # Verbose is required to pass to the callback manager
            )

    return llm

EMBEDDINGS = HuggingFaceInstructEmbeddings(model_name=EMBEDDING_MODEL_NAME, 
                                            model_kwargs={"device": DEVICE_TYPE})

DB = FAISS.load_local(PERSIST_DIRECTORY, EMBEDDINGS)

RETRIEVER = DB.as_retriever(search_kwargs={'k': 16})

template = """\

 



{context}








{question}

"""

LLM = load_model()#load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME)

QA_CHAIN_PROMPT = PromptTemplate.from_template(template)

QA = ConversationalRetrievalChain.from_llm(LLM, retriever=RETRIEVER,
                                           combine_docs_chain_kwargs={"prompt": QA_CHAIN_PROMPT})
```

The app.py file has this:
```

from fastapi import FastAPI
from langserve import add_routes

from chain import QA

app = FastAPI(title="Retrieval App")

add_routes(app, QA)

if __name__ == "__main__":
    import uvicorn

    uvicorn.run(app, host="0.0.0.0", port=8000)`

However when running /stream route, I get this warning:

NFO:     Started server process [491117]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
INFO:     127.0.0.1:48024 - "GET /docs HTTP/1.1" 200 OK
INFO:     127.0.0.1:48024 - "GET /openapi.json HTTP/1.1" 200 OK
INFO:     127.0.0.1:53430 - "POST /stream HTTP/1.1" 200 OK
/home/zeeshan/miniconda3/envs/llama/lib/python3.10/site-packages/langchain/llms/llamacpp.py:352: RuntimeWarning: coroutine 'AsyncCallbackManagerForLLMRun.on_llm_new_token' was never awaited
  run_manager.on_llm_new_token(
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
```
And the output is like this:

![image](https://github.com/langchain-ai/langserve/assets/93662126/8e1ac812-b2e7-4df0-a0e9-ca70f4e1ee40)

This seems not to be streaming output tokens. Does langserve even support langchain llamacpp or I am doing something wrong?