drop deprecated code
Browse files- src/extraction_normal.py +5 -6
- src/transformation.py +0 -3
- src/utils.py +0 -9
src/extraction_normal.py
CHANGED
@@ -56,14 +56,14 @@ def organize_page_data(url: str = BASE_URL ,product= None):
|
|
56 |
|
57 |
|
58 |
def get_soup_by_url(url, product: str = None):
|
59 |
-
proxy = generate_proxy_url()
|
60 |
-
proxies = {'http': proxy,
|
61 |
-
|
62 |
if product is None:
|
63 |
url = url
|
64 |
else:
|
65 |
url = f'{url}/{product}'
|
66 |
-
r = requests.get(url=url
|
67 |
s = BeautifulSoup(r.content, 'html.parser')
|
68 |
return s
|
69 |
|
@@ -104,5 +104,4 @@ def get_location(s):
|
|
104 |
|
105 |
|
106 |
if __name__ == '__main__':
|
107 |
-
data = main(product='carros', pages=1)
|
108 |
-
print(data)
|
|
|
56 |
|
57 |
|
58 |
def get_soup_by_url(url, product: str = None):
|
59 |
+
# proxy = generate_proxy_url()
|
60 |
+
# proxies = {'http': proxy,
|
61 |
+
# 'https': proxy}
|
62 |
if product is None:
|
63 |
url = url
|
64 |
else:
|
65 |
url = f'{url}/{product}'
|
66 |
+
r = requests.get(url=url)
|
67 |
s = BeautifulSoup(r.content, 'html.parser')
|
68 |
return s
|
69 |
|
|
|
104 |
|
105 |
|
106 |
if __name__ == '__main__':
|
107 |
+
data = main(product='carros', pages=1)
|
|
src/transformation.py
DELETED
@@ -1,3 +0,0 @@
|
|
1 |
-
import pandas as pd
|
2 |
-
from extraction import *
|
3 |
-
|
|
|
|
|
|
|
|
src/utils.py
CHANGED
@@ -16,15 +16,6 @@ def timer_decorator(func):
|
|
16 |
return result
|
17 |
return wrapper
|
18 |
|
19 |
-
def async_timer_decorator(func):
|
20 |
-
async def wrapper(*args, **kwargs):
|
21 |
-
start_time = time.time()
|
22 |
-
result = await func(*args, **kwargs)
|
23 |
-
end_time = time.time()
|
24 |
-
print(f"Function {func.__name__} took {end_time - start_time:.4f} seconds to execute")
|
25 |
-
return result
|
26 |
-
return wrapper
|
27 |
-
|
28 |
|
29 |
def generate_proxy_url():
|
30 |
port = random.randint(10000,10099)
|
|
|
16 |
return result
|
17 |
return wrapper
|
18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
19 |
|
20 |
def generate_proxy_url():
|
21 |
port = random.randint(10000,10099)
|