replaced chromedriver/selenium with zenrows

web_search.py

@@ -35,19 +35,19 @@ def search(msg, query_phrase):
         # query_phrase, keywords = ut.get_search_phrase_and_keywords(msg, [])
         google_text = ""
         try:
-            logger.info(f"asking google {msg}; rephrased: {query_phrase}")
+            print(f"asking google {msg}; rephrased: {query_phrase}")
             google_text, urls_all, urls_used, tried_index, urls_tried = search_google(msg, query_phrase)
         except:
             traceback.print_exc()
 
-        logger.info("\n\nFinal response: ")
+        print("\n\nFinal response: ")
 
         for item in google_text:
-            logger.info(
+            print(
                 f"\n##############################################################################################\nSource: {item['source']}"
             )
-            logger.info(f"{item['text']}")
-            logger.info(f"URL: {item['url']}")
+            print(f"{item['text']}")
+            print(f"URL: {item['url']}")
         return google_text
     except KeyboardInterrupt:
         traceback.print_exc()
@@ -88,6 +88,7 @@ def process_url(url, timeout):
                 # response = driver.page_source
                 client = ZenRowsClient(os.getenv('zenrows_api_key'))
                 response = client.get(url)
+                print(f'got response, status: {response.status_code}')
                 # result = response_text_extract(url=url, response=response)
                 result = response.text
             except Exception:
@@ -101,16 +102,16 @@ def process_url(url, timeout):
             #     return "", url
     except Exception:
         traceback.print_exc()
-        logger.info(f"{site} err")
+        print(f"{site} err")
         pass
-    logger.info(f"Processed {site}: {len(response.text)} / {len(result)} {int((time.time() - start_time) * 1000)} ms")
+    print(f"Processed {site}: {len(response.text)} / {len(result)} {int((time.time() - start_time) * 1000)} ms")
     return result, url
 
 
 def process_urls(urls):
     # Create a ThreadPoolExecutor with 5 worker threads
     response = []
-    logger.info("entering process urls")
+    print("entering process urls")
     full_text = ""
     used_index = 0
     urls_used = ["" for i in range(30)]
@@ -139,7 +140,7 @@ def process_urls(urls):
             urls_tried[tried_index] = url
             tried_index += 1
             urls.remove(url)
-            logger.info(f"queued {ut.extract_site(url)}, {timeout}")
+            print(f"queued {ut.extract_site(url)}, {timeout}")
             # Process the responses as they arrive
             # for future in in_process:
             #     if future.done():
@@ -149,7 +150,7 @@ def process_urls(urls):
             if len(result) > 0:
                 urls_used[used_index] = url
                 used_index += 1
-                logger.info(
+                print(
                     f"adding {len(result)} chars from {ut.extract_site(url)} to {len(response)} prior responses"
                 )
                 if "an error has occurred" not in result.lower() and "permission to view this page" not in result.lower() and "403 ERROR" not in result.lower() and "have been blocked" not in result.lower() and "too many requests" not in result.lower():
@@ -163,7 +164,7 @@ def process_urls(urls):
 
         if (len(urls) == 0 and len(in_process) == 0) or (time.time() - start_time > 28):
             # executor.shutdown(wait=False)
-            logger.info(
+            print(
                 f"n****** exiting process urls early {len(response)} {int(time.time() - start_time)} secs\n"
             )
             return response, used_index, urls_used, tried_index, urls_tried
@@ -171,7 +172,7 @@ def process_urls(urls):
     except:
         traceback.print_exc()
     # executor.shutdown(wait=False)
-    logger.info(
+    print(
         f"\n*****processed all urls {len(response)}  {int(time.time() - start_time)} secs"
     )
     return response, urls_used, tried_index, urls_tried
@@ -182,11 +183,11 @@ def extract_subtext(text):
 
 
 def request_google(query_phrase):
-    logger.info(f"***** search {query_phrase}")
+    print(f"***** search {query_phrase}")
     sort = "&sort=date-sdate:d:w"
     if "today" in query_phrase or "latest" in query_phrase:
         sort = "&sort=date-sdate:d:s"
-    # logger.info(f"search for: {query_phrase}")
+    print(f"search for: {query_phrase}")
     google_query = en.quote(query_phrase)
     response = []
     try:
@@ -203,14 +204,14 @@ def request_google(query_phrase):
         )
         response = requests.get(url)
         response_json = json.loads(response.text)
-        logger.info(f"***** google search {int((time.time() - start_wall_time) * 10) / 10} sec")
+        print(f"***** google search {int((time.time() - start_wall_time) * 10) / 10} sec")
     except:
         traceback.print_exc()
         return []
 
     # see if we got anything useful from Google
     if "items" not in response_json.keys():
-        logger.info("no return from google ...", response, response_json.keys())
+        print("no return from google ...", response, response_json.keys())
         return []
 
     urls = []
@@ -224,43 +225,43 @@ def request_google(query_phrase):
     return urls
 
 
-def response_text_extract(url, response):
-    extract_text = ""
-    if url.endswith("pdf"):
-        pass
-    else:
-        if response is not None:
-            elements = partition_html(text=response)
-            str_elements = []
-            logger.info('\n***** elements')
-            for e in elements:
-                stre = str(e).replace("  ", " ")
-                str_elements.append(stre)
-            extract_text = ''.join(extract_subtext(str_elements))
-            logger.info(
-                f"***** unstructured found {len(elements)} elements, {sum([len(str(e)) for e in elements])} raw chars, {len(extract_text)} extract"
-            )
-
-    if len(extract_text.strip()) < 8:
-        return ""
-    else:
-        return extract_text
+# def response_text_extract(url, response):
+#     extract_text = ""
+#     if url.endswith("pdf"):
+#         pass
+#     else:
+#         if response is not None:
+#             elements = partition_html(text=response)
+#             str_elements = []
+#             logger.info('\n***** elements')
+#             for e in elements:
+#                 stre = str(e).replace("  ", " ")
+#                 str_elements.append(stre)
+#             extract_text = ''.join(extract_subtext(str_elements))
+#             logger.info(
+#                 f"***** unstructured found {len(elements)} elements, {sum([len(str(e)) for e in elements])} raw chars, {len(extract_text)} extract"
+#             )
+#
+#     if len(extract_text.strip()) < 8:
+#         return ""
+#     else:
+#         return extract_text
 
 
-def extract_items_from_numbered_list(text):
-    items = ""
-    elements = text.split("\n")
-    for candidate in elements:
-        candidate = candidate.lstrip(". \t")
-        if len(candidate) > 4 and candidate[0].isdigit():
-            candidate = candidate[1:].lstrip(". ")
-            if (
-                    len(candidate) > 4 and candidate[0].isdigit()
-            ):  # strip second digit if more than 10 items
-                candidate = candidate[1:].lstrip(". ")
-            logger.info("E {}".format(candidate))
-            items += candidate + " "
-    return items
+# def extract_items_from_numbered_list(text):
+#     items = ""
+#     elements = text.split("\n")
+#     for candidate in elements:
+#         candidate = candidate.lstrip(". \t")
+#         if len(candidate) > 4 and candidate[0].isdigit():
+#             candidate = candidate[1:].lstrip(". ")
+#             if (
+#                     len(candidate) > 4 and candidate[0].isdigit()
+#             ):  # strip second digit if more than 10 items
+#                 candidate = candidate[1:].lstrip(". ")
+#             logger.info("E {}".format(candidate))
+#             items += candidate + " "
+#     return items
 
 
 def search_google(original_query, query_phrase):
@@ -295,8 +296,8 @@ def search_google(original_query, query_phrase):
         # initialize scan of Google urls
         start_wall_time = time.time()
         full_text, urls_used, tried_index, urls_tried = process_urls(all_urls)
-        logger.info(f"***** urls_processed {int((time.time() - start_wall_time) * 10) / 10} sec")
-        logger.info("return from url processsing")
+        print(f"***** urls_processed {int((time.time() - start_wall_time) * 10) / 10} sec")
+        print("return from url processsing")
     except:
         traceback.print_exc()
     return full_text, all_urls, urls_used, tried_index, urls_tried