Update extract.py
Browse files- extract.py +17 -17
extract.py
CHANGED
@@ -6,9 +6,12 @@ from selenium.common.exceptions import WebDriverException
|
|
6 |
from bs4 import BeautifulSoup
|
7 |
import time
|
8 |
import random
|
9 |
-
import
|
10 |
from fp.fp import FreeProxy
|
11 |
|
|
|
|
|
|
|
12 |
# Lista de User Agents para rotar
|
13 |
user_agents = [
|
14 |
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
@@ -18,22 +21,8 @@ user_agents = [
|
|
18 |
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
|
19 |
"Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
|
20 |
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36",
|
21 |
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:87.0) Gecko/20100101 Firefox/87.0",
|
22 |
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0",
|
23 |
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:88.0) Gecko/20100101 Firefox/88.0",
|
24 |
-
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0",
|
25 |
-
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
|
26 |
-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
|
27 |
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36",
|
28 |
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
|
29 |
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
|
30 |
-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
|
31 |
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:91.0) Gecko/20100101 Firefox/91.0",
|
32 |
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0",
|
33 |
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:91.0) Gecko/20100101 Firefox/91.0",
|
34 |
]
|
35 |
|
36 |
-
|
37 |
def get_random_user_agent():
|
38 |
return random.choice(user_agents)
|
39 |
|
@@ -58,6 +47,7 @@ def extract_data(user_input, mode):
|
|
58 |
options.add_argument(f"user-agent={get_random_user_agent()}")
|
59 |
options.add_argument('--proxy-server=%s' % proxy_url)
|
60 |
|
|
|
61 |
try:
|
62 |
wd = webdriver.Chrome(options=options)
|
63 |
window_size = get_random_window_size()
|
@@ -65,6 +55,9 @@ def extract_data(user_input, mode):
|
|
65 |
|
66 |
# Construir la URL de búsqueda
|
67 |
url_busqueda = f"https://app.neilpatel.com/es/traffic_analyzer/keywords?domain={user_input}&lang=es&locId=2724&mode={mode}"
|
|
|
|
|
|
|
68 |
wd.get(url_busqueda)
|
69 |
|
70 |
# Espera aleatoria para simular el comportamiento humano
|
@@ -73,8 +66,15 @@ def extract_data(user_input, mode):
|
|
73 |
# Obtener el contenido de la página
|
74 |
page_content = wd.page_source
|
75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
76 |
except WebDriverException as e:
|
77 |
-
|
78 |
return []
|
79 |
finally:
|
80 |
if wd:
|
@@ -132,4 +132,4 @@ def extract_data(user_input, mode):
|
|
132 |
|
133 |
# Parsear el texto filtrado
|
134 |
datos_parseados = parsear_texto(lineas_filtradas)
|
135 |
-
return datos_parseados
|
|
|
6 |
from bs4 import BeautifulSoup
|
7 |
import time
|
8 |
import random
|
9 |
+
import logging
|
10 |
from fp.fp import FreeProxy
|
11 |
|
12 |
+
# Configurar el logging
|
13 |
+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
14 |
+
|
15 |
# Lista de User Agents para rotar
|
16 |
user_agents = [
|
17 |
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
|
|
21 |
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
|
22 |
"Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
|
23 |
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
]
|
25 |
|
|
|
26 |
def get_random_user_agent():
|
27 |
return random.choice(user_agents)
|
28 |
|
|
|
47 |
options.add_argument(f"user-agent={get_random_user_agent()}")
|
48 |
options.add_argument('--proxy-server=%s' % proxy_url)
|
49 |
|
50 |
+
wd = None
|
51 |
try:
|
52 |
wd = webdriver.Chrome(options=options)
|
53 |
window_size = get_random_window_size()
|
|
|
55 |
|
56 |
# Construir la URL de búsqueda
|
57 |
url_busqueda = f"https://app.neilpatel.com/es/traffic_analyzer/keywords?domain={user_input}&lang=es&locId=2724&mode={mode}"
|
58 |
+
|
59 |
+
logging.info(f"Making request to {url_busqueda} with IP: {proxy_url}")
|
60 |
+
|
61 |
wd.get(url_busqueda)
|
62 |
|
63 |
# Espera aleatoria para simular el comportamiento humano
|
|
|
66 |
# Obtener el contenido de la página
|
67 |
page_content = wd.page_source
|
68 |
|
69 |
+
# Obtener el código de respuesta HTTP
|
70 |
+
response_status = wd.execute_script("return document.readyState")
|
71 |
+
if response_status == "complete":
|
72 |
+
logging.info(f"Request with IP: {proxy_url} returned status code 200")
|
73 |
+
else:
|
74 |
+
logging.warning(f"Request with IP: {proxy_url} did not return status code 200")
|
75 |
+
|
76 |
except WebDriverException as e:
|
77 |
+
logging.error(f"Request failed with proxy {proxy_url}. Error: {e}")
|
78 |
return []
|
79 |
finally:
|
80 |
if wd:
|
|
|
132 |
|
133 |
# Parsear el texto filtrado
|
134 |
datos_parseados = parsear_texto(lineas_filtradas)
|
135 |
+
return datos_parseados
|