Update app.py

app.py

@@ -227,7 +227,8 @@ import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse, urljoin
 
-def get_links_from_page(url, visited_urls, domain_links):
+
+def get_links_from_page(url, visited_urls, all_links):
     if url in visited_urls:
         return
 
@@ -246,15 +247,13 @@ def get_links_from_page(url, visited_urls, domain_links):
         for link in links:
             href = link.get('href')
             absolute_url = urljoin(base_url, href)
-            parsed_url = urlparse(absolute_url)
-
-            if parsed_url.netloc == urlparse(url).netloc:
-                domain_links.add(absolute_url)
-                get_links_from_page(absolute_url, visited_urls, domain_links)
+            all_links.add(absolute_url)
+            get_links_from_page(absolute_url, visited_urls, all_links)
 
     else:
         print(f"Failed to retrieve content from {url}. Status code: {response.status_code}")
 
+
 def get_all_links_from_domain(domain_url):
     visited_urls = set()
     domain_links = set()
@@ -288,7 +287,7 @@ fe_app = gr.ChatInterface(
 # load the model asynchronously on startup and save it into memory 
 @app.on_event("startup")
 async def startup():
-    domain_url = 'https://globl.contact/'
+    domain_url = 'https://www.bofrost.de/faq/'
     links = get_all_links_from_domain(domain_url)
     print("Links from the domain:", links)