import uuid import json from os import makedirs, remove from os.path import exists, dirname from bs4 import BeautifulSoup import requests ''' - Error handing - Look if alternative to main tag is needed. Provide error message if main tag is not found. - Menus are li tags with a tags within. - li tags with text and tags should be exported - Find divs that have text or p tags maybe other tags like divs - Export the text ''' def get_url_content( url ): file_path = 'page-content/' + uuid.uuid5( uuid.NAMESPACE_URL, url ).hex + '.json' makedirs(dirname(file_path), exist_ok=True) if exists( file_path ): with open( file_path, 'r' ) as file: strings = json.load( file ) else: try: strings = extract_strings( url ) except Exception as exception: raise exception with open( file_path, 'w' ) as file: json.dump( strings, file ) return strings def extract_strings( url ): try : soup = get_soup( url ) except Exception as exception: raise exception if soup is None: raise Exception('No HTML content found.') # Remove scripts and styles. for script in soup(["script", "style"]): script.decompose() content = get_main_content( soup ) if content is None : raise Exception('No main content found.') strings = get_tags_text( content ) if strings is None : raise Exception('No text found.') return strings # Make request and get html content. def get_soup( url ): file_path = 'web-pages/' + uuid.uuid5( uuid.NAMESPACE_URL, url ).hex + '.html' makedirs(dirname(file_path), exist_ok=True) if exists( file_path ): with open( file_path, 'r' ) as web_page: html = web_page.read() else: headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'} response = requests.get( url, headers=headers ) response.raise_for_status() if not response.text: raise Exception('HTML empty.') html = response.text with open( file_path, 'w' ) as file: file.write( html ) return BeautifulSoup(html, 'html.parser') def get_main_content( soup ): content = soup.find( "div", { "class": "post-body" } ) if content is not None: print('Has .post-body class.') return content content = soup.find( "div", { "class": "article-content" } ) if content is not None: print('Has .article-content class.') return content content = soup.find( "div", { "class": "blog-post-content" } ) if content is not None: print('Has .blog-post-content class.') return content content = soup.find( "div", { "class": "region-content" } ) if content is not None: print('Has .region-content class.') return content content = soup.find( "div", { "class": "entry-content" } ) if content is not None: print('Has .entry-content class.') return content content = soup.find( "div", { "class": "region--content" } ) if content is not None: print('Has .region--content class.') return content content = soup.find( "div", { "class": "article" } ) if content is not None: print('Has .article class.') return content content = soup.find( "div", { "class": "article-inner_html" } ) if content is not None: print('Has .article-inner_html class.') return content content = soup.find( "div", { "id": "bmdDetail-Content" } ) if content is not None: print('Has .bmdDetail-Content id.') return content content = soup.find( "div", { "id": "main" } ) if content is not None: print('Has .bmdDetail-Content id.') return content content = soup.main if content is not None: print('Has main tag.') return content content = soup.find( "article" ) if content is not None: print('Has article tag.') return content content = soup.find( "body" ) if content is not None: print('Has body tag.') return content return None def get_tags_text( soup ): text = [] tags = soup.find_all( allowed_tags ) for tag in tags: if tag.name == 'div' : for div in tag.find_all(text=True, recursive=False): found_text = div.get_text( ' ', strip=True ) if found_text != '': text.append( found_text ) else : found_text = tag.get_text( ' ', strip=True ) if found_text != '': text.append( found_text ) return text def allowed_tags( tag ): return tag.name == 'li' or tag.name == 'p' or tag.name == 'h1' or tag.name == 'h2' or tag.name == 'h3' or tag.name == 'span' or tag.name == 'div' # -------------------------------------- # # Extract content from main tag. def get_main( soup ): return soup.main def get_deepest_divs( tag ): # Get all the divs from within a tag. return [div for div in tag.findAll('div') if not div.find('div')] def get_tag_text( tags ): text = '' for tag in tags: print(tag.find_all('li')) # text += [p.get_text() for p in tag.find_all('p)] return text def get_list_text( tags ): list_items = [] for tag in tags: list_items = tag.find_all(find_direct_text) return list_items def find_div_text( tag ): return tag.name == 'div' and tag.find( text=True, recursive=False ) and tag.find( text=True, recursive=False ).strip() if __name__ == '__main__': url = 'https://www.cms.gov/Medicare/Billing/ElectronicBillingEDITrans' print(extract_content(url))