| import re | |
| import streamlit as st | |
| import textract | |
| import tempfile | |
| import random | |
| from pathlib import Path | |
| import spacy | |
| from spacy.tokens import DocBin | |
| import srsly | |
| from spacy.matcher import Matcher | |
| st.title('Index and Search a Collection of Documents') | |
| def download_model(select_model:str): | |
| with st.spinner(f'Loading model {select_model}'): | |
| spacy.cli.download(select_model) | |
| return True | |
| doc_bin = DocBin() | |
| models = srsly.read_json('models.json') | |
| models[''] = [] #require the user to choose a language | |
| languages = models.keys() | |
| language = st.selectbox("Language", languages, index=len(models.keys())-1, help="Select the language of your materials.") | |
| if language: | |
| select_model = st.selectbox("Model", models[language], help="spaCy model") | |
| if select_model: | |
| model_downloaded = download_model(select_model) | |
| if model_downloaded: | |
| nlp = spacy.load(select_model) | |
| nlp.max_length = 1200000 | |
| uploaded_files = st.file_uploader("Select files to process", accept_multiple_files=True) | |
| for uploaded_file in uploaded_files: | |
| file_type = uploaded_file.type | |
| temp = tempfile.NamedTemporaryFile() | |
| temp.write(uploaded_file.getvalue()) | |
| try: | |
| text = textract.process(temp.name) | |
| text = text.decode('utf-8') | |
| doc = nlp(text) | |
| st.write(text) | |
| except Exception as e: | |
| st.error(e) | |
| #st.download_button('Download', '', 'text/plain') | |