tiiuae

/

falcon-40b

like 2.4k

I am using langchain to load falcon-40b on an H100 GPU machine but I get this and nothing is generated when I pass a context to it using FAISS

The model 'RWForCausalLM' is not supported for text-generation. Supported models are
 ['BartForCausalLM', 'BertLMHeadModel', 'BertGenerationDecoder', 'BigBirdForCausalLM', 'BigBirdPegasusForCausalLM', 'BioGptForCausalLM', 

'BlenderbotForCausalLM', 'BlenderbotSmallForCausalLM', 'BloomForCausalLM', 'CamembertForCausalLM', 'CodeGenForCausalLM', 'CpmAntForCausalLM', 

'CTRLLMHeadModel', 'Data2VecTextForCausalLM', 'ElectraForCausalLM', 'ErnieForCausalLM', 'GitForCausalLM', 'GPT2LMHeadModel', 'GPT2LMHeadModel', 'GPTBigCodeForCausalLM', 'GPTNeoForCausalLM', 'GPTNeoXForCausalLM', 'GPTNeoXJapaneseForCausalLM', 'GPTJForCausalLM', 'LlamaForCausalLM', 

'MarianForCausalLM', 'MBartForCausalLM', 'MegaForCausalLM', 'MegatronBertForCausalLM', 'MvpForCausalLM', 'OpenLlamaForCausalLM', 'OpenAIGPTLMHeadModel', 'OPTForCausalLM', 'PegasusForCausalLM', 'PLBartForCausalLM', 'ProphetNetForCausalLM', 'QDQBertLMHeadModel', 

'ReformerModelWithLMHead', 'RemBertForCausalLM', 'RobertaForCausalLM', 'RobertaPreLayerNormForCausalLM', 'RoCBertForCausalLM', 
'RoFormerForCausalLM', 'RwkvForCausalLM', 'Speech2Text2ForCausalLM', 'TransfoXLLMHeadModel', 'TrOCRForCausalLM', 'XGLMForCausalLM', 

'XLMWithLMHeadModel', 'XLMProphetNetForCausalLM', 'XLMRobertaForCausalLM', 'XLMRobertaXLForCausalLM', 'XLNetLMHeadModel', 'XmodForCausalLM'].

This is how I am loading the model and providing FAISS embeddings to it

def load_embeddings(sotre_name, path):
    with open(f"{path}/faiss_{sotre_name}.pkl", "rb") as f:
        VectorStore = pickle.load(f)
    return VectorStore

Embedding_store_path = f"./dbfs"


# hf_embed = load_embeddings(sotre_name='huggingface_fm_lambdalabs_faiss', 
hf_embed = load_embeddings(sotre_name='store_template', 
                                    path=Embedding_store_path)

def get_similar_docs(question, similar_doc_count):
  return hf_embed.similarity_search(question, k=similar_doc_count)

def build_qa_chain():
  torch.cuda.empty_cache()
  model_name = "tiiuae/falcon-40b"
 
  tokenizer = AutoTokenizer.from_pretrained(model_name)
  instruct_pipeline = pipeline(model=model_name, tokenizer=tokenizer, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto", 
                               return_full_text=True, max_new_tokens=256, top_p=0.95, top_k=50)
 
  # Defining our prompt content.
  # langchain will load our similar documents as {context}
  template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
 
  Instruction: 
  You are an experienced in .. and your job is to help providing the best answer related to .... 
  Use only information in the following paragraphs to answer the question at the end. Explain the answer with reference to these paragraphs. If you don't know, say that you do not know.
 
  {context}
 
  Question: {question}
 
  Response:
  """
  prompt = PromptTemplate(input_variables=['context', 'question'], template=template)
 
  hf_pipe = HuggingFacePipeline(pipeline=instruct_pipeline)
  # Set verbose=True to see the full prompt:
  return load_qa_chain(llm=hf_pipe, chain_type="stuff", prompt=prompt, verbose=True)

qa_chain = build_qa_chain()

def answer_question(question):
  similar_docs = get_similar_docs(question, similar_doc_count=1)
  result = qa_chain({"input_documents": similar_docs, "question": question})
  
  print("question: " + question)
  print(" ")
  print("Answer: ")
  print(result['output_text'])
  print(" ")  
  print("Sources")
  print(" ")
  for d in result["input_documents"]:
    source_id = d.metadata["source"]
    print(d.page_content)
    print("Source " + source_id)
    print(" ")
    
answer_question("<question>?")
while True:
    query = input("\nEnter a query: ")
    if query == "exit":
        break

    # Get the answer from the chain
    answer_question(query)