Jorge Henao commited on
Commit
63b3ab3
β€’
1 Parent(s): 105521b

refactor en multiples archivos

Browse files
app.py CHANGED
@@ -1,93 +1,19 @@
1
- from haystack.nodes import BM25Retriever, FARMReader
2
- from haystack.document_stores import ElasticsearchDocumentStore
3
- from haystack.pipelines import ExtractiveQAPipeline
4
  from abc import ABC, abstractmethod
5
- import certifi
6
  import gradio as gr
7
  import examples
 
 
 
8
 
9
- ca_certs=certifi.where()
10
-
11
-
12
- class Config():
13
- es_host = "ask2democracy.es.us-central1.gcp.cloud.es.io"
14
- es_user = "elastic"
15
- es_password = "siKAHmmk2flwEaKNqQVZwp49"
16
- proposals_index = "petrolfo"
17
- #reader_model_name_or_path = "deepset/roberta-base-squad2"
18
- reader_model_name_or_path = "deepset/xlm-roberta-large-squad2"
19
- use_gpu = True
20
-
21
- class DocumentQueries(ABC):
22
-
23
- @abstractmethod
24
- def search_by_query(self, query : str, retriever_top_k: int, reader_top_k: int, es_index: str):
25
- pass
26
-
27
- class ExtractiveProposalQueries(DocumentQueries):
28
-
29
- def __init__(self, es_host: str, es_index: str, es_user, es_password, reader_name_or_path: str, use_gpu = True) -> None:
30
- reader = FARMReader(model_name_or_path = reader_name_or_path, use_gpu = use_gpu, num_processes=1)
31
- self._initialize_pipeline(es_host, es_index, es_user, es_password, reader = reader)
32
-
33
-
34
- def _initialize_pipeline(self, es_host, es_index, es_user, es_password, reader = None):
35
- if reader is not None:
36
- self.reader = reader
37
- self.es_host = es_host
38
- self.es_user = es_user
39
- self.es_password = es_password
40
- self.document_store = ElasticsearchDocumentStore(host = es_host, username=es_user, password=es_password, index = es_index, port = 443, scheme='https', verify_certs=True, ca_certs=ca_certs)
41
- self.retriever = BM25Retriever(document_store = self.document_store)
42
- self.pipe = ExtractiveQAPipeline(self.reader, self.retriever)
43
-
44
- def search_by_query(self, query : str, retriever_top_k: int, reader_top_k: int, es_index: str = None) :
45
- if es_index is not None:
46
- self._initialize_pipeline(self.es_host, es_index, self.es_user, self.es_password)
47
- params = {"Retriever": {"top_k": retriever_top_k}, "Reader": {"top_k": reader_top_k}}
48
- prediction = self.pipe.run( query = query, params = params)
49
- return prediction["answers"]
50
-
51
- query = ExtractiveProposalQueries(es_host = Config.es_host, es_index = Config.proposals_index,
52
- es_user = Config.es_user, es_password = Config.es_password,
53
- reader_name_or_path = Config.reader_model_name_or_path,
54
- use_gpu = Config.use_gpu)
55
-
56
- def update(query):
57
- return f"{query}", f"{query}", f"{query}", f"{query}"
58
-
59
- def search(question):
60
- p1_result = query.search_by_query(query = question, retriever_top_k = 5, reader_top_k = 1, es_index = "petro")
61
- p2_result = query.search_by_query(query = question, retriever_top_k = 5, reader_top_k = 1, es_index = "rodolfo")
62
-
63
- return [p1_result[0].answer,
64
- p1_result[0].context,
65
- p2_result[0].answer,
66
- p2_result[0].context]
67
-
68
- def update(query):
69
- return f"{query}", f"{query}", f"{query}", f"{query}"
70
-
71
- def search(question):
72
- p1_result = query.search_by_query(query = question, retriever_top_k = 5, reader_top_k = 1, es_index = "petro")
73
- p2_result = query.search_by_query(query = question, retriever_top_k = 5, reader_top_k = 1, es_index = "rodolfo")
74
-
75
- return [p1_result[0].answer,
76
- p1_result[0].context,
77
- p2_result[0].answer,
78
- p2_result[0].context]
79
-
80
  query = ExtractiveProposalQueries(es_host = Config.es_host, es_index = Config.proposals_index,
81
  es_user = Config.es_user, es_password = Config.es_password,
82
  reader_name_or_path = Config.reader_model_name_or_path,
83
  use_gpu = Config.use_gpu)
84
 
85
- def update(query):
86
- return f"{query}", f"{query}", f"{query}", f"{query}"
87
-
88
  def search(question):
89
- p1_result = query.search_by_query(query = question, retriever_top_k = 5, reader_top_k = 3, es_index = "petro")
90
- p2_result = query.search_by_query(query = question, retriever_top_k = 5, reader_top_k = 3, es_index = "rodolfo")
91
 
92
  return [p1_result[0].answer,
93
  p1_result[0].context,
@@ -161,7 +87,7 @@ examples = [
161
  ]
162
 
163
  iface = gr.Interface(fn=search,
164
- title= "Ask2Democracy πŸ‡¨πŸ‡΄ - Elecciones precidenciales Colombia 2022",
165
  description = description,
166
  inputs=[
167
  gr.inputs.Textbox(lines=2, label="hazle una pregunta a los candidatos [Abajo algunos ejemplos]πŸ‘‡", placeholder='que va a hacer con la deuda del ICETEX?')
 
 
 
 
1
  from abc import ABC, abstractmethod
 
2
  import gradio as gr
3
  import examples
4
+ from document_quieries import ExtractiveProposalQueries
5
+ from config import Config
6
+ import certifi
7
 
8
+ ca_certs=certifi.where()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  query = ExtractiveProposalQueries(es_host = Config.es_host, es_index = Config.proposals_index,
10
  es_user = Config.es_user, es_password = Config.es_password,
11
  reader_name_or_path = Config.reader_model_name_or_path,
12
  use_gpu = Config.use_gpu)
13
 
 
 
 
14
  def search(question):
15
+ p1_result = query.search_by_query(query = question, retriever_top_k = 2, reader_top_k = 1, es_index = "petro")
16
+ p2_result = query.search_by_query(query = question, retriever_top_k = 2, reader_top_k = 1, es_index = "rodolfo")
17
 
18
  return [p1_result[0].answer,
19
  p1_result[0].context,
 
87
  ]
88
 
89
  iface = gr.Interface(fn=search,
90
+ title= "PregΓΊntale a la democracia πŸ‡¨πŸ‡΄ - Elecciones precidenciales Colombia 2022",
91
  description = description,
92
  inputs=[
93
  gr.inputs.Textbox(lines=2, label="hazle una pregunta a los candidatos [Abajo algunos ejemplos]πŸ‘‡", placeholder='que va a hacer con la deuda del ICETEX?')
config.py CHANGED
@@ -1,3 +1,4 @@
 
1
  class Config():
2
  es_host = "ask2democracy.es.us-central1.gcp.cloud.es.io"
3
  es_user = "elastic"
 
1
+
2
  class Config():
3
  es_host = "ask2democracy.es.us-central1.gcp.cloud.es.io"
4
  es_user = "elastic"
document_quieries β†’ document_quieries.py RENAMED
@@ -1,12 +1,8 @@
 
1
  from haystack.nodes import BM25Retriever, FARMReader
2
  from haystack.document_stores import ElasticsearchDocumentStore
3
  from haystack.pipelines import ExtractiveQAPipeline
4
 
5
- import certifi
6
- ca_certs=certifi.where()
7
-
8
- from abc import ABC, abstractmethod
9
-
10
  class DocumentQueries(ABC):
11
 
12
  @abstractmethod
@@ -15,11 +11,10 @@ class DocumentQueries(ABC):
15
 
16
  class ExtractiveProposalQueries(DocumentQueries):
17
 
18
- def __init__(self, es_host: str, es_index: str, es_user, es_password, reader_name_or_path: str, use_gpu = False) -> None:
19
  reader = FARMReader(model_name_or_path = reader_name_or_path, use_gpu = use_gpu, num_processes=1)
20
  self._initialize_pipeline(es_host, es_index, es_user, es_password, reader = reader)
21
 
22
-
23
  def _initialize_pipeline(self, es_host, es_index, es_user, es_password, reader = None):
24
  if reader is not None:
25
  self.reader = reader
@@ -36,4 +31,3 @@ class ExtractiveProposalQueries(DocumentQueries):
36
  params = {"Retriever": {"top_k": retriever_top_k}, "Reader": {"top_k": reader_top_k}}
37
  prediction = self.pipe.run( query = query, params = params)
38
  return prediction["answers"]
39
-
 
1
+ from abc import ABC, abstractmethod
2
  from haystack.nodes import BM25Retriever, FARMReader
3
  from haystack.document_stores import ElasticsearchDocumentStore
4
  from haystack.pipelines import ExtractiveQAPipeline
5
 
 
 
 
 
 
6
  class DocumentQueries(ABC):
7
 
8
  @abstractmethod
 
11
 
12
  class ExtractiveProposalQueries(DocumentQueries):
13
 
14
+ def __init__(self, es_host: str, es_index: str, es_user, es_password, reader_name_or_path: str, use_gpu = True) -> None:
15
  reader = FARMReader(model_name_or_path = reader_name_or_path, use_gpu = use_gpu, num_processes=1)
16
  self._initialize_pipeline(es_host, es_index, es_user, es_password, reader = reader)
17
 
 
18
  def _initialize_pipeline(self, es_host, es_index, es_user, es_password, reader = None):
19
  if reader is not None:
20
  self.reader = reader
 
31
  params = {"Retriever": {"top_k": retriever_top_k}, "Reader": {"top_k": reader_top_k}}
32
  prediction = self.pipe.run( query = query, params = params)
33
  return prediction["answers"]