ramortegui commited on
Commit
869e15e
1 Parent(s): 59277db

Remove blank lines

Browse files
Files changed (1) hide show
  1. app.py +0 -14
app.py CHANGED
@@ -8,24 +8,17 @@ from transformers import AutoTokenizer
8
 
9
  bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
10
  data = bshtml_dir_loader.load()
11
- print("loading documents")
12
 
13
  bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
14
 
15
- print("add tokenizer")
16
 
17
  text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
18
  chunk_size=100,
19
  chunk_overlap=0,
20
  separator="\n")
21
 
22
-
23
- print("Add text spliters")
24
-
25
  documents = text_splitter.split_documents(data)
26
 
27
- print("Getting HF embeddings")
28
-
29
  embeddings = HuggingFaceEmbeddings()
30
 
31
  llm = HuggingFacePipeline.from_model_id(
@@ -33,24 +26,17 @@ llm = HuggingFacePipeline.from_model_id(
33
  task="text-generation",
34
  model_kwargs={"temperature" : 0, "max_length" : 500})
35
 
36
- print("Adding LLM hugginFacePipeline with bigscience bloomz")
37
 
38
  vectordb = Chroma.from_documents(documents=documents, embedding=embeddings)
39
 
40
- print("Getting vectors")
41
-
42
  doc_retriever = vectordb.as_retriever()
43
 
44
- print("Creating Retreiver")
45
-
46
 
47
  shakespeare_qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=doc_retriever)
48
 
49
- print("Add shakespeare qa")
50
 
51
  def query(query):
52
  shakespeare_qa.run(query)
53
-
54
 
55
  iface = gr.Interface(fn=query, inputs="text", outputs="text")
56
  iface.launch()
 
8
 
9
  bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
10
  data = bshtml_dir_loader.load()
 
11
 
12
  bloomz_tokenizer = AutoTokenizer.from_pretrained("bigscience/bloomz-1b7")
13
 
 
14
 
15
  text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(bloomz_tokenizer,
16
  chunk_size=100,
17
  chunk_overlap=0,
18
  separator="\n")
19
 
 
 
 
20
  documents = text_splitter.split_documents(data)
21
 
 
 
22
  embeddings = HuggingFaceEmbeddings()
23
 
24
  llm = HuggingFacePipeline.from_model_id(
 
26
  task="text-generation",
27
  model_kwargs={"temperature" : 0, "max_length" : 500})
28
 
 
29
 
30
  vectordb = Chroma.from_documents(documents=documents, embedding=embeddings)
31
 
 
 
32
  doc_retriever = vectordb.as_retriever()
33
 
 
 
34
 
35
  shakespeare_qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=doc_retriever)
36
 
 
37
 
38
  def query(query):
39
  shakespeare_qa.run(query)
 
40
 
41
  iface = gr.Interface(fn=query, inputs="text", outputs="text")
42
  iface.launch()