Spaces:
Configuration error
Configuration error
File size: 1,447 Bytes
4ea8f5c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import requests
from bs4 import BeautifulSoup
def scrape(url):
URL = url
headers = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"}
# Here the user agent is for Edge browser on windows 10. You can find your browser user agent from the above given link.
r = requests.get(url=URL, headers=headers)
if r.status_code ==200:
# print(r.content)
soup = BeautifulSoup(r.content, 'html.parser') # If this line causes an error, run 'pip install html5lib' or install html5lib
# print(soup.prettify())
# b = soup.find_all('p','ul li')
# for yy in b:
# print(yy.get_text())
flag = 0;
enter = """
"""
str = ""
body = soup.body.find_all()
for tag in body:
if tag.name=='p' or tag.name=='h1' or tag.name=='h2' or tag.name=='h3' :
str += tag.get_text()
str += enter
if tag.name=='p':
flag=1
elif (tag.name == 'ul' or tag.name=='ol') and flag==1:
liBody=tag.find_all('li')
for text in liBody:
str+=text.get_text()
str += enter
# print(str)
# print(soup.body.get_text())
return str
else:
return "" |