Save strings as json instead of text

Allows for better processing of sentences

README.md

@@ -39,6 +39,7 @@ google_search_engine_id = "search-engine-id"
 - [ ] Improve fetched content.
   - [ ] Get some content from every search result.
   - [ ] Find sentences that contain the search keywords.
+  - [ ] Div's with text & tags. Extract text from tags and then decompose the tags.
 - [ ] Summarization requires truncation. Find solution where not needed.
 - [ ] Support German content.
 - [ ] Improve queries to include more keywords (Expand abrivations & define context)

beautiful_soup/beautiful_soup.py

@@ -1,4 +1,5 @@
 import uuid
+import json
 from os import makedirs, remove
 from os.path import exists, dirname
 from bs4 import BeautifulSoup
@@ -14,22 +15,22 @@ import requests
 '''
 
 def get_url_content( url ):
-    file_path = 'page-content/' + uuid.uuid5( uuid.NAMESPACE_URL, url ).hex + '.txt'
-    makedirs(dirname(file_path), exist_ok=True)
-    if exists( file_path ):
-        with open( file_path, 'r' ) as file_content:
-            content = file_content.read()
+    file = 'page-content/' + uuid.uuid5( uuid.NAMESPACE_URL, url ).hex + '.json'
+    makedirs(dirname(file), exist_ok=True)
+    if exists( file ):
+        with open( file, 'r' ) as file_content:
+            strings = json.load( file )
     else:
         try:
-            content = extract_content( url )
+            strings = extract_strings( url )
         except Exception as exception:
             raise exception
-        with open( file_path, 'w' ) as file:
-            file.write( content.strip() )
+        with open( file, 'w' ) as file:
+            json.dump( strings, file )
 
-    return content
+    return strings
 
-def extract_content( url ):
+def extract_strings( url ):
     try :
         soup = get_soup( url )
     except Exception as exception:
@@ -46,11 +47,10 @@ def extract_content( url ):
     if content is None :
         raise Exception('No main content found.')
 
-    text = get_tags_text( content )
-    if text is None :
+    strings = get_tags_text( content )
+    if strings is None :
         raise Exception('No text found.')
-
-    return text
+    return strings
 
 # Make request and get html content.
 def get_soup( url ):
@@ -136,16 +136,16 @@ def get_main_content( soup ):
     return None
 
 def get_tags_text( soup ):
-    text = ''
+    text = []
     tags = soup.find_all( allowed_tags )
     for tag in tags:
         if tag.name == 'div' :
             for div in tag.find_all(text=True, recursive=False):
                 found_text = div.get_text( ' ', strip=True )
                 if found_text != '':
-                    text += found_text
+                    text.append( found_text )
         else :
-            text += tag.get_text( ' ', strip=True ) + ' '
+            text.append( tag.get_text( ' ', strip=True ))
     return text
 
 def allowed_tags( tag ):