Spaces:
Runtime error
Runtime error
from bs4 import BeautifulSoup as bs | |
import requests | |
from typing import Dict, List, Optional | |
from fake_http_header import FakeHttpHeader | |
class Scraper: | |
def __init__(self): | |
... | |
def get_product(self, url : str) -> Dict: | |
webpage = requests.get(url, headers=FakeHttpHeader().as_header_dict()) | |
f = open('webpage_out.html','w') | |
f.write(webpage.content.decode()) | |
f.close() | |
if webpage.status_code != 200: | |
return 'Error Loading Link' | |
try: | |
webpage = bs(webpage.content) | |
title = webpage.findAll("span", attrs={"id": 'productTitle'})[0].text.strip() | |
categories = [x.strip().lower() for x in webpage.findAll("div", attrs={"id": 'wayfinding-breadcrumbs_feature_div'})[0].text.strip().split('\n') if x.strip()!='' and len(x.strip()) >=3] | |
desc = webpage.findAll("div", attrs={"id": 'featurebullets_feature_div'})[0].text.replace('About this item','').strip() | |
except IndexError as e: | |
if webpage.content.find('captcha')!=-1: | |
return {'description' : 'Detected as a Bot. Please Try Again Later. Till then, you can continue to type in your description, or manually copy from Amazon.'} | |
return {'description' : f'{title}\n{desc}', 'labels' : categories} |