Spaces:
Running
Running
| # Created by Leandro Carneiro at 19/01/2024 | |
| # Description: | |
| # ------------------------------------------------ | |
| import os.path | |
| import time | |
| from googleapiclient.discovery import build | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import constants | |
| def google_search_api(search_term, api_key, cse_id, **kwargs): | |
| try: | |
| service = build("customsearch", "v1", developerKey=api_key) | |
| res = service.cse().list(q=search_term, cx=cse_id, **kwargs).execute() | |
| return res['items'] | |
| except Exception as e: | |
| return -1 | |
| def search_google(subject, sites): | |
| try: | |
| results = [] | |
| for site in sites: | |
| print(' Buscando notícias no domínio: ' + site) | |
| query = f"{subject} site:{site}" | |
| sites_searched = google_search_api(query, os.environ['GOOGLE_KEY'], os.environ['GOOGLE_SEARCH'], num=constants.num_sites) | |
| if sites_searched == -1: | |
| results.append(site) | |
| else: | |
| for s in sites_searched: | |
| if 'pdf' not in s['link'].lower(): | |
| results.append(s['link']) | |
| else: | |
| print(' Arquivo PDF encontrado: ' + s['link']) | |
| #time.sleep(3) | |
| print(' Total de sites encontrados: ' + str(len(results))) | |
| return results | |
| except Exception as e: | |
| print(str(e)) | |
| return str(e) | |
| def retrieve_text_from_site(sites): | |
| result = [] | |
| for site in sites: | |
| print(' Baixando texto do site: ' + site) | |
| try: | |
| response = requests.get(site) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| result.append(soup.get_text()) | |
| except Exception as e: | |
| result.append('Erro na recuperação do texto: ' + str(e)) | |
| return result | |
| def delete_base(local_base): | |
| try: | |
| for i in os.listdir(local_base): | |
| file_path = os.path.join(local_base, i) | |
| os.remove(file_path) | |
| return 0 | |
| except Exception as e: | |
| return str(e) | |
| def save_on_base(sites, texts, local_base): | |
| try: | |
| for i in range(len(sites)): | |
| filename = f'news{i}.txt' | |
| with open(os.path.join(local_base, filename), 'w', encoding='utf-8') as file: | |
| file.write(texts[i]) | |
| with open(os.path.join(local_base, 'filename_url.csv'), 'a', encoding='utf-8') as file: | |
| file.write(filename + ';' + sites[i] + '\n') | |
| return 0 | |
| except Exception as e: | |
| return str(e) | |