wilmars commited on
Commit
80c5d93
1 Parent(s): d78091b

drop deprecated code

Browse files
src/extraction_normal.py CHANGED
@@ -56,14 +56,14 @@ def organize_page_data(url: str = BASE_URL ,product= None):
56
 
57
 
58
  def get_soup_by_url(url, product: str = None):
59
- proxy = generate_proxy_url()
60
- proxies = {'http': proxy,
61
- 'https': proxy}
62
  if product is None:
63
  url = url
64
  else:
65
  url = f'{url}/{product}'
66
- r = requests.get(url=url, proxies=proxies)
67
  s = BeautifulSoup(r.content, 'html.parser')
68
  return s
69
 
@@ -104,5 +104,4 @@ def get_location(s):
104
 
105
 
106
  if __name__ == '__main__':
107
- data = main(product='carros', pages=1)
108
- print(data)
 
56
 
57
 
58
  def get_soup_by_url(url, product: str = None):
59
+ # proxy = generate_proxy_url()
60
+ # proxies = {'http': proxy,
61
+ # 'https': proxy}
62
  if product is None:
63
  url = url
64
  else:
65
  url = f'{url}/{product}'
66
+ r = requests.get(url=url)
67
  s = BeautifulSoup(r.content, 'html.parser')
68
  return s
69
 
 
104
 
105
 
106
  if __name__ == '__main__':
107
+ data = main(product='carros', pages=1)
 
src/transformation.py DELETED
@@ -1,3 +0,0 @@
1
- import pandas as pd
2
- from extraction import *
3
-
 
 
 
 
src/utils.py CHANGED
@@ -16,15 +16,6 @@ def timer_decorator(func):
16
  return result
17
  return wrapper
18
 
19
- def async_timer_decorator(func):
20
- async def wrapper(*args, **kwargs):
21
- start_time = time.time()
22
- result = await func(*args, **kwargs)
23
- end_time = time.time()
24
- print(f"Function {func.__name__} took {end_time - start_time:.4f} seconds to execute")
25
- return result
26
- return wrapper
27
-
28
 
29
  def generate_proxy_url():
30
  port = random.randint(10000,10099)
 
16
  return result
17
  return wrapper
18
 
 
 
 
 
 
 
 
 
 
19
 
20
  def generate_proxy_url():
21
  port = random.randint(10000,10099)