corpora/sourcer.py

import requests
from string import Template
from bs4 import BeautifulSoup
from dotenv import load_dotenv
from os import getenv

load_dotenv()

google_key = getenv('GOOGLE_KEY')
google_engine = getenv('GOOGLE_ENGINE')

url = Template(f'https://www.googleapis.com/customsearch/v1?key={google_key}&cx={google_engine}&q=$query')

def search_web(query: str) -> list:
    query = '+'.join(query.split())
    results = requests.get(url.substitute(query=query)).json()['items']

    links = [item['link'] for item in results]

    texts = []
    for link in links:
        resp = requests.get(link)
        soup = BeautifulSoup(resp.text, 'html.parser')
        text = []
        # remove lists
        for tag in soup.find_all('li'):
            tag.extract()
        
        tags = soup.find_all('p')
        for tag in tags:
            text.append(tag.text)
        texts.append('\n'.join(text))

    return texts