Create visit_webpage_tool.py

visit_webpage_tool.py

@@ -0,0 +1,35 @@
+from transformers import Tool
+import requests
+from markdownify import markdownify as md
+from requests.exceptions import RequestException
+import re
+
+class VisitWebpageTool(Tool):
+    name = "visit_webpage"
+    description = "Visits a webpage at the given URL and returns its content as a markdown string."
+    inputs = {
+        "url": {
+            "type": "text",
+            "description": "The URL of the webpage to visit.",
+        }
+    }
+    output_type = "text"
+
+    def forward(self, url: str) -> str:
+        try:
+            # Send a GET request to the URL
+            response = requests.get(url)
+            response.raise_for_status()  # Raise an exception for bad status codes
+
+            # Convert the HTML content to Markdown
+            markdown_content = md(response.text).strip()
+
+            # Remove multiple line breaks
+            markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
+
+            return markdown_content
+
+        except RequestException as e:
+            return f"Error fetching the webpage: {str(e)}"
+        except Exception as e:
+            return f"An unexpected error occurred: {str(e)}"