| from .common import process_file |
| from langchain.document_loaders import UnstructuredHTMLLoader |
| import requests |
| import re |
| import unicodedata |
| import tempfile |
| import os |
| import streamlit as st |
| from streamlit.runtime.uploaded_file_manager import UploadedFileRec, UploadedFile |
|
|
| def process_html(vector_store, file, stats_db): |
| return process_file(vector_store, file, UnstructuredHTMLLoader, ".html", stats_db=stats_db) |
|
|
|
|
| def get_html(url): |
| response = requests.get(url) |
| if response.status_code == 200: |
| return response.text |
| else: |
| return None |
|
|
| def create_html_file(url, content): |
| file_name = slugify(url) + ".html" |
| temp_file_path = os.path.join(tempfile.gettempdir(), file_name) |
| with open(temp_file_path, 'w') as temp_file: |
| temp_file.write(content) |
|
|
| record = UploadedFileRec(id=None, name=file_name, type='text/html', data=open(temp_file_path, 'rb').read()) |
| uploaded_file = UploadedFile(record) |
| |
| return uploaded_file, temp_file_path |
|
|
| def delete_tempfile(temp_file_path, url, ret): |
| try: |
| os.remove(temp_file_path) |
| if ret: |
| st.write(f"✅ Content saved... {url} ") |
| except OSError as e: |
| print(f"Error while deleting the temporary file: {str(e)}") |
| if ret: |
| st.write(f"❌ Error while saving content... {url} ") |
|
|
| def slugify(text): |
| text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8') |
| text = re.sub(r'[^\w\s-]', '', text).strip().lower() |
| text = re.sub(r'[-\s]+', '-', text) |
| return text |
