SemSup-XC / fetch_prod.py
Pranjal2041's picture
Initial Commit
4014562
raw
history blame
1.31 kB
from bs4 import BeautifulSoup as bs
import requests
from typing import Dict, List, Optional
from fake_http_header import FakeHttpHeader
class Scraper:
def __init__(self):
...
def get_product(self, url : str) -> Dict:
webpage = requests.get(url, headers=FakeHttpHeader().as_header_dict())
f = open('webpage_out.html','w')
f.write(webpage.content.decode())
f.close()
if webpage.status_code != 200:
return 'Error Loading Link'
try:
webpage = bs(webpage.content)
title = webpage.findAll("span", attrs={"id": 'productTitle'})[0].text.strip()
categories = [x.strip().lower() for x in webpage.findAll("div", attrs={"id": 'wayfinding-breadcrumbs_feature_div'})[0].text.strip().split('\n') if x.strip()!='' and len(x.strip()) >=3]
desc = webpage.findAll("div", attrs={"id": 'featurebullets_feature_div'})[0].text.replace('About this item','').strip()
except IndexError as e:
if webpage.content.find('captcha')!=-1:
return {'description' : 'Detected as a Bot. Please Try Again Later. Till then, you can continue to type in your description, or manually copy from Amazon.'}
return {'description' : f'{title}\n{desc}', 'labels' : categories}