awacke1 commited on
Commit
2de1374
1 Parent(s): f351c98

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -6
app.py CHANGED
@@ -1,9 +1,9 @@
1
  import streamlit as st
2
  import requests
3
- from bs4 import BeautifulSoup
4
  import os
5
  import urllib
6
  import base64
 
7
 
8
  EXCLUDED_FILES = ['app.py', 'requirements.txt', 'pre-requirements.txt', 'packages.txt', 'README.md','.gitattributes', "backup.py","Dockerfile"]
9
 
@@ -22,16 +22,13 @@ def download_file(url, local_filename):
22
  def download_html_and_files(url):
23
  html_content = requests.get(url).text
24
  soup = BeautifulSoup(html_content, 'html.parser')
25
-
26
  base_url = urllib.parse.urlunparse(urllib.parse.urlparse(url)._replace(path='', params='', query='', fragment=''))
27
-
28
  for link in soup.find_all('a'):
29
  file_url = urllib.parse.urljoin(base_url, link.get('href'))
30
  local_filename = urllib.parse.urlparse(file_url).path.split('/')[-1]
31
  if local_filename:
32
  link['href'] = local_filename
33
  download_file(file_url, local_filename)
34
-
35
  with open("index.html", "w") as file:
36
  file.write(str(soup))
37
 
@@ -52,8 +49,8 @@ def show_download_links():
52
  st.sidebar.markdown(get_download_link(file), unsafe_allow_html=True)
53
 
54
  def main():
55
- st.sidebar.title('Bulk Download Tool')
56
- url = st.sidebar.text_input('Please enter a URL to bulk download text and files')
57
  if st.sidebar.button('📥 Get All the Content'):
58
  download_html_and_files(url)
59
  show_download_links()
 
1
  import streamlit as st
2
  import requests
 
3
  import os
4
  import urllib
5
  import base64
6
+ from bs4 import BeautifulSoup
7
 
8
  EXCLUDED_FILES = ['app.py', 'requirements.txt', 'pre-requirements.txt', 'packages.txt', 'README.md','.gitattributes', "backup.py","Dockerfile"]
9
 
 
22
  def download_html_and_files(url):
23
  html_content = requests.get(url).text
24
  soup = BeautifulSoup(html_content, 'html.parser')
 
25
  base_url = urllib.parse.urlunparse(urllib.parse.urlparse(url)._replace(path='', params='', query='', fragment=''))
 
26
  for link in soup.find_all('a'):
27
  file_url = urllib.parse.urljoin(base_url, link.get('href'))
28
  local_filename = urllib.parse.urlparse(file_url).path.split('/')[-1]
29
  if local_filename:
30
  link['href'] = local_filename
31
  download_file(file_url, local_filename)
 
32
  with open("index.html", "w") as file:
33
  file.write(str(soup))
34
 
 
49
  st.sidebar.markdown(get_download_link(file), unsafe_allow_html=True)
50
 
51
  def main():
52
+ st.sidebar.title('Web Bulk Downloader')
53
+ url = st.sidebar.text_input('Please enter a Web URL to bulk download text and files')
54
  if st.sidebar.button('📥 Get All the Content'):
55
  download_html_and_files(url)
56
  show_download_links()