Arafath10 commited on
Commit
81b8336
1 Parent(s): 5f74ea1

Update scraper.py

Browse files
Files changed (1) hide show
  1. scraper.py +3 -3
scraper.py CHANGED
@@ -22,7 +22,7 @@ class Scraper:
22
  await page.goto(url)
23
 
24
  # Get the title
25
- #title = await page.title()
26
 
27
  # Get all links
28
  page_url = await page.evaluate("""() => {
@@ -37,7 +37,7 @@ class Scraper:
37
 
38
 
39
  await browser.close()
40
- return page_url, page_content
41
 
42
  @staticmethod
43
  async def power_scrapper(url):
@@ -112,5 +112,5 @@ class Scraper:
112
 
113
  return {"title": title, "URL": links, "Content": text_content}
114
  except:
115
- links, text_content = await Scraper.power_scrapper_2(url)
116
  return {"title": title, "URL": links, "Content": text_content}
 
22
  await page.goto(url)
23
 
24
  # Get the title
25
+ title = await page.title()
26
 
27
  # Get all links
28
  page_url = await page.evaluate("""() => {
 
37
 
38
 
39
  await browser.close()
40
+ return title,page_url, page_content
41
 
42
  @staticmethod
43
  async def power_scrapper(url):
 
112
 
113
  return {"title": title, "URL": links, "Content": text_content}
114
  except:
115
+ title,links, text_content = await Scraper.power_scrapper_2(url)
116
  return {"title": title, "URL": links, "Content": text_content}