Spaces:
Runtime error
Runtime error
import requests | |
from bs4 import BeautifulSoup | |
from swarms.tools.base import BaseToolSet, tool | |
from swarms.utils.logger import logger | |
class RequestsGet(BaseToolSet): | |
def get(self, url: str) -> str: | |
"""Run the tool.""" | |
html = requests.get(url).text | |
soup = BeautifulSoup(html) | |
non_readable_tags = soup.find_all( | |
["script", "style", "header", "footer", "form"] | |
) | |
for non_readable_tag in non_readable_tags: | |
non_readable_tag.extract() | |
content = soup.get_text("\n", strip=True) | |
if len(content) > 300: | |
content = content[:300] + "..." | |
logger.debug( | |
f"\nProcessed RequestsGet, Input Url: {url} " f"Output Contents: {content}" | |
) | |
return content | |