Update ✨Entity Linking Application✨.py

✨Entity Linking Application✨.py

@@ -13,10 +13,18 @@ import streamlit as st
 import time
 from openai import OpenAI
 import sys
-#from googlesearch import search
 import time
+from bs4 import BeautifulSoup
+from fake_useragent import UserAgent
+import requests
 
 
+ua = UserAgent()
+
+headers = {
+    "User-Agent": f"{ua.random}"
+    }
+
 folder_path = '/home/user/app/qids_folder'
 
 if not os.path.exists(folder_path):
@@ -72,11 +80,14 @@ async def mains(name, single, combi):
     qids = set()
     
     async with aiohttp.ClientSession() as session:
-        url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={name}&srlimit=10&srprop=&srenablerewrites=True&srinfo=suggestion&format=json"
-        json_suggestion = await fetch_json(url, session)
-        results = json_suggestion.get('query', {}).get('search')
-        for i in results:
-            data.add(i.get('title'))
+        url = f"https://www.google.com/search?q={name} site:en.wikipedia.org inurl:/wiki/ -inurl:? -inurl:Category: -inurl:File: -inurl:Special: -inurl:Help:"
+        html = requests.get(url, headers=headers)
+        soup = BeautifulSoup(html.text, "html.parser")
+        elements_with_href = soup.find_all(href=True)
+        href_links = [element['href'] for element in elements_with_href]
+        for link in href_links:
+            if link.startswith('https://en.wikipedia.org/wiki/'):
+                data.add(link.split("/")[-1])
 
         wikipedia_url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={name}&srlimit=1&srprop=&srenablerewrites=True&srinfo=suggestion&format=json"
         json_data = await fetch_json(wikipedia_url, session)