pseudotensor commited on
Commit
935bf6f
1 Parent(s): ca9b3d4

Update with h2oGPT hash 8513731040927b5a455110324918954b09460b05

Browse files
Files changed (2) hide show
  1. src/gen.py +10 -3
  2. src/serpapi.py +167 -0
src/gen.py CHANGED
@@ -779,6 +779,10 @@ def main(
779
  # always allow DISABLED
780
  if LangChainMode.DISABLED.value not in langchain_modes:
781
  langchain_modes.append(LangChainMode.DISABLED.value)
 
 
 
 
782
 
783
  # update
784
  langchain_mode_paths = str_to_dict(langchain_mode_paths)
@@ -1036,7 +1040,7 @@ def main(
1036
  print(f"Generating model with params:\n{locals_print}", flush=True)
1037
  print("Command: %s\nHash: %s" % (str(' '.join(sys.argv)), git_hash), flush=True)
1038
 
1039
- if langchain_mode != "Disabled":
1040
  # SECOND PLACE where LangChain referenced, but all imports are kept local so not required
1041
  from gpt_langchain import prep_langchain, get_some_dbs_from_hf, get_persist_directory
1042
  if is_hf:
@@ -1233,7 +1237,8 @@ def main(
1233
  **all_kwargs))
1234
  score_model_state0 = dict(model=smodel, tokenizer=stokenizer, device=sdevice,
1235
  base_model=score_model, tokenizer_base_model='', lora_weights='',
1236
- inference_server='', prompt_type='', prompt_dict='')
 
1237
 
1238
  if enable_captions:
1239
  if pre_load_caption_model:
@@ -1244,7 +1249,9 @@ def main(
1244
  else:
1245
  caption_loader = False
1246
 
1247
- if pre_load_embedding_model and langchain_mode != 'Disabled' and not use_openai_embedding:
 
 
1248
  from src.gpt_langchain import get_embedding
1249
  hf_embedding_model = dict(name=hf_embedding_model,
1250
  model=get_embedding(use_openai_embedding, hf_embedding_model=hf_embedding_model,
 
779
  # always allow DISABLED
780
  if LangChainMode.DISABLED.value not in langchain_modes:
781
  langchain_modes.append(LangChainMode.DISABLED.value)
782
+ if not have_langchain:
783
+ # only allow disabled, not even LLM that is langchain related
784
+ langchain_mode = LangChainMode.DISABLED.value
785
+ langchain_modes = [langchain_mode]
786
 
787
  # update
788
  langchain_mode_paths = str_to_dict(langchain_mode_paths)
 
1040
  print(f"Generating model with params:\n{locals_print}", flush=True)
1041
  print("Command: %s\nHash: %s" % (str(' '.join(sys.argv)), git_hash), flush=True)
1042
 
1043
+ if langchain_mode != LangChainMode.DISABLED.value:
1044
  # SECOND PLACE where LangChain referenced, but all imports are kept local so not required
1045
  from gpt_langchain import prep_langchain, get_some_dbs_from_hf, get_persist_directory
1046
  if is_hf:
 
1237
  **all_kwargs))
1238
  score_model_state0 = dict(model=smodel, tokenizer=stokenizer, device=sdevice,
1239
  base_model=score_model, tokenizer_base_model='', lora_weights='',
1240
+ inference_server='', prompt_type='', prompt_dict='',
1241
+ visible_models=None, h2ogpt_key=None)
1242
 
1243
  if enable_captions:
1244
  if pre_load_caption_model:
 
1249
  else:
1250
  caption_loader = False
1251
 
1252
+ if pre_load_embedding_model and \
1253
+ langchain_mode != LangChainMode.DISABLED.value and \
1254
+ not use_openai_embedding:
1255
  from src.gpt_langchain import get_embedding
1256
  hf_embedding_model = dict(name=hf_embedding_model,
1257
  model=get_embedding(use_openai_embedding, hf_embedding_model=hf_embedding_model,
src/serpapi.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import functools
2
+ import typing
3
+
4
+ import aiohttp
5
+ from langchain.docstore.document import Document
6
+ from langchain import SerpAPIWrapper
7
+
8
+ from src.utils_langchain import _chunk_sources, add_parser, _add_meta
9
+ from urllib.parse import urlparse
10
+
11
+
12
+ class H2OSerpAPIWrapper(SerpAPIWrapper):
13
+ def get_search_documents(self, query,
14
+ query_action=True,
15
+ chunk=True, chunk_size=512,
16
+ db_type='chroma',
17
+ headsize=50,
18
+ top_k_docs=-1):
19
+ docs = self.run(query, headsize)
20
+
21
+ chunk_sources = functools.partial(_chunk_sources, chunk=chunk, chunk_size=chunk_size, db_type=db_type)
22
+ docs = chunk_sources(docs)
23
+
24
+ # choose chunk type
25
+ if query_action:
26
+ docs = [x for x in docs if x.metadata['chunk_id'] >= 0]
27
+ else:
28
+ docs = [x for x in docs if x.metadata['chunk_id'] == -1]
29
+
30
+ # get score assuming search results scale with ranking
31
+ delta = 0.05
32
+ [x.metadata.update(score=0.1 + delta * x.metadata['chunk_id'] if x.metadata['chunk_id'] >= 0 else -1) for x in
33
+ docs]
34
+
35
+ # ensure see all results up to cutoff or mixing with non-web docs
36
+ if top_k_docs >= 1:
37
+ top_k_docs = max(top_k_docs, len(docs))
38
+
39
+ return docs, top_k_docs
40
+
41
+ async def arun(self, query: str, headsize: int, **kwargs: typing.Any) -> list:
42
+ """Run query through SerpAPI and parse result async."""
43
+ return self._process_response(await self.aresults(query), query, headsize)
44
+
45
+ def run(self, query: str, headsize: int, **kwargs: typing.Any) -> list:
46
+ """Run query through SerpAPI and parse result."""
47
+ return self._process_response(self.results(query), query, headsize)
48
+
49
+ @staticmethod
50
+ def _process_response(res: dict, query: str, headsize: int) -> list:
51
+ try:
52
+ return H2OSerpAPIWrapper.__process_response(res, query, headsize)
53
+ except Exception as e:
54
+ print("SERP search failed: %s" % str(e))
55
+ return []
56
+
57
+ @staticmethod
58
+ def __process_response(res: dict, query: str, headsize: int) -> list:
59
+ docs = []
60
+
61
+ res1 = SerpAPIWrapper._process_response(res)
62
+ if res1:
63
+ if isinstance(res1, str) and not res1.startswith('['): # avoid snippets
64
+ docs += [Document(page_content='Web search result %s: ' % len(docs) + res1,
65
+ metadata=dict(source='Web Search %s for %s' % (len(docs), query), score=0.0))]
66
+ elif isinstance(res1, list):
67
+ for x in res1:
68
+ date = ''
69
+ content = ''
70
+ if 'source' in x:
71
+ source = x['source']
72
+ content += '%s says' % source
73
+ else:
74
+ content = 'Web search result %s: ' % len(docs)
75
+ if 'date' in x:
76
+ date = x['date']
77
+ content += ' %s' % date
78
+ if 'title' in x:
79
+ content += ': %s' % x['title']
80
+ if 'snippet' in x:
81
+ content += ': %s' % x['snippet']
82
+ if 'link' in x:
83
+ link = x['link']
84
+ domain = urlparse(link).netloc
85
+ font_size = 2
86
+ source_name = domain
87
+ http_content = """<font size="%s"><a href="%s" target="_blank" rel="noopener noreferrer">%s</a></font>""" % (
88
+ font_size, link, source_name)
89
+ source = 'Web Search %s' % len(docs) + \
90
+ ' from Date: %s Domain: %s Link: %s' % (date, domain, http_content)
91
+ if date:
92
+ content += ' around %s' % date
93
+ content += ' according to %s' % domain
94
+ else:
95
+ source = 'Web Search %s for %s' % (len(docs), query)
96
+ docs += [Document(page_content=content, metadata=dict(source=source, score=0.0))]
97
+
98
+ if "knowledge_graph" in res.keys():
99
+ knowledge_graph = res["knowledge_graph"]
100
+ title = knowledge_graph["title"] if "title" in knowledge_graph else ""
101
+ if "description" in knowledge_graph.keys():
102
+ docs += [Document(page_content='Web search result %s: ' % len(docs) + knowledge_graph["description"],
103
+ metadata=dict(source='Web Search %s with knowledge_graph description for %s' % (
104
+ len(docs), query), score=0.0))]
105
+ for key, value in knowledge_graph.items():
106
+ if (
107
+ type(key) == str
108
+ and type(value) == str
109
+ and key not in ["title", "description"]
110
+ and not key.endswith("_stick")
111
+ and not key.endswith("_link")
112
+ and not value.startswith("http")
113
+ ):
114
+ docs += [Document(page_content='Web search result %s: ' % len(docs) + f"{title} {key}: {value}.",
115
+ metadata=dict(
116
+ source='Web Search %s with knowledge_graph for %s' % (len(docs), query),
117
+ score=0.0))]
118
+ if "organic_results" in res.keys():
119
+ for org_res in res["organic_results"]:
120
+ keys_to_try = ['snippet', 'snippet_highlighted_words', 'rich_snippet', 'rich_snippet_table', 'link']
121
+ for key in keys_to_try:
122
+ if key in org_res.keys():
123
+ date = ''
124
+ domain = ''
125
+ link = ''
126
+ snippet1 = ''
127
+ if key != 'link':
128
+ snippet1 = org_res[key]
129
+ if 'date' in org_res.keys():
130
+ date = org_res['date']
131
+ snippet1 += ' on %s' % date
132
+ else:
133
+ date = 'unknown date'
134
+ if 'link' in org_res.keys():
135
+ link = org_res['link']
136
+ domain = urlparse(link).netloc
137
+ if key == 'link':
138
+ # worst case, only url might have REST info
139
+ snippet1 += ' Link at %s: <a href="%s">%s</a>' % (domain, link, domain)
140
+ else:
141
+ snippet1 += ' according to %s' % domain
142
+ if snippet1:
143
+ font_size = 2
144
+ source_name = domain
145
+ http_content = """<font size="%s"><a href="%s" target="_blank" rel="noopener noreferrer">%s</a></font>""" % (
146
+ font_size, link, source_name)
147
+ source = 'Web Search %s' % len(docs) + \
148
+ ' from Date: %s Domain: %s Link: %s' % (date, domain, http_content)
149
+ domain_simple = domain.replace('www.', '').replace('.com', '')
150
+ snippet1 = '%s says on %s: %s' % (domain_simple, date, snippet1)
151
+ docs += [Document(page_content=snippet1, metadata=dict(source=source), score=0.0)]
152
+ break
153
+ if "buying_guide" in res.keys():
154
+ docs += [Document(page_content='Web search result %s: ' % len(docs) + res["buying_guide"],
155
+ metadata=dict(source='Web Search %s with buying_guide for %s' % (len(docs), query)),
156
+ score=0.0)]
157
+ if "local_results" in res.keys() and "places" in res["local_results"].keys():
158
+ docs += [Document(page_content='Web search result %s: ' % len(docs) + res["local_results"]["places"],
159
+ metadata=dict(
160
+ source='Web Search %s with local_results_places for %s' % (len(docs), query)),
161
+ score=0.0)]
162
+
163
+ # add meta
164
+ add_meta = functools.partial(_add_meta, headsize=headsize, parser='SERPAPI')
165
+ add_meta(docs, query)
166
+
167
+ return docs