Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| import spacy | |
| import wikipediaapi | |
| import wikipedia | |
| from wikipedia.exceptions import DisambiguationError | |
| from transformers import TFAutoModel, AutoTokenizer | |
| import numpy as np | |
| import pandas as pd | |
| import faiss | |
| import datetime | |
| import time | |
| try: | |
| nlp = spacy.load("en_core_web_sm") | |
| except: | |
| spacy.cli.download("en_core_web_sm") | |
| nlp = spacy.load("en_core_web_sm") | |
| wh_words = ['what', 'who', 'how', 'when', 'which'] | |
| def get_concepts(text): | |
| text = text.lower() | |
| doc = nlp(text) | |
| concepts = [] | |
| for chunk in doc.noun_chunks: | |
| if chunk.text not in wh_words: | |
| concepts.append(chunk.text) | |
| return concepts | |
| def get_passages(text, k=100): | |
| doc = nlp(text) | |
| passages = [] | |
| passage_len = 0 | |
| passage = "" | |
| sents = list(doc.sents) | |
| for i in range(len(sents)): | |
| sen = sents[i] | |
| passage_len += len(sen) | |
| if passage_len >= k: | |
| passages.append(passage) | |
| passage = sen.text | |
| passage_len = len(sen) | |
| continue | |
| elif i == (len(sents) - 1): | |
| passage += " " + sen.text | |
| passages.append(passage) | |
| passage = "" | |
| passage_len = 0 | |
| continue | |
| passage += " " + sen.text | |
| return passages | |
| def get_dicts_for_dpr(concepts, n_results=20, k=100): | |
| dicts = [] | |
| for concept in concepts: | |
| wikis = wikipedia.search(concept, results=n_results) | |
| st.write(f"{concept} No of Wikis: {len(wikis)}") | |
| for wiki in wikis: | |
| try: | |
| html_page = wikipedia.page(title=wiki, auto_suggest=False) | |
| except DisambiguationError: | |
| continue | |
| htmlResults = html_page.content | |
| passages = get_passages(htmlResults, k=k) | |
| for passage in passages: | |
| i_dicts = {} | |
| i_dicts['text'] = passage | |
| i_dicts['title'] = wiki | |
| dicts.append(i_dicts) | |
| return dicts | |
| passage_encoder = TFAutoModel.from_pretrained("nlpconnect/dpr-ctx_encoder_bert_uncased_L-2_H-128_A-2") | |
| query_encoder = TFAutoModel.from_pretrained("nlpconnect/dpr-question_encoder_bert_uncased_L-2_H-128_A-2") | |
| p_tokenizer = AutoTokenizer.from_pretrained("nlpconnect/dpr-ctx_encoder_bert_uncased_L-2_H-128_A-2") | |
| q_tokenizer = AutoTokenizer.from_pretrained("nlpconnect/dpr-question_encoder_bert_uncased_L-2_H-128_A-2") | |
| def get_title_text_combined(passage_dicts): | |
| res = [] | |
| for p in passage_dicts: | |
| res.append(tuple((p['title'], p['text']))) | |
| return res | |
| def extracted_passage_embeddings(processed_passages, max_length=156): | |
| passage_inputs = p_tokenizer.batch_encode_plus( | |
| processed_passages, | |
| add_special_tokens=True, | |
| truncation=True, | |
| padding="max_length", | |
| max_length=max_length, | |
| return_token_type_ids=True | |
| ) | |
| passage_embeddings = passage_encoder.predict([np.array(passage_inputs['input_ids']), np.array(passage_inputs['attention_mask']), | |
| np.array(passage_inputs['token_type_ids'])], | |
| batch_size=64, | |
| verbose=1) | |
| return passage_embeddings | |
| def extracted_query_embeddings(queries, max_length=64): | |
| query_inputs = q_tokenizer.batch_encode_plus( | |
| queries, | |
| add_special_tokens=True, | |
| truncation=True, | |
| padding="max_length", | |
| max_length=max_length, | |
| return_token_type_ids=True | |
| ) | |
| query_embeddings = query_encoder.predict([np.array(query_inputs['input_ids']), | |
| np.array(query_inputs['attention_mask']), | |
| np.array(query_inputs['token_type_ids'])], | |
| batch_size=1, | |
| verbose=1) | |
| return query_embeddings | |
| def get_pagetext(page): | |
| s = str(page).replace("/t","") | |
| return s | |
| def get_wiki_summary(search): | |
| wiki_wiki = wikipediaapi.Wikipedia('en') | |
| page = wiki_wiki.page(search) | |
| def get_wiki_summaryDF(search): | |
| wiki_wiki = wikipediaapi.Wikipedia('en') | |
| page = wiki_wiki.page(search) | |
| isExist = page.exists() | |
| if not isExist: | |
| return isExist, "Not found", "Not found", "Not found", "Not found" | |
| pageurl = page.fullurl | |
| pagetitle = page.title | |
| pagesummary = page.summary[0:60] | |
| pagetext = get_pagetext(page.text) | |
| backlinks = page.backlinks | |
| linklist = "" | |
| for link in backlinks.items(): | |
| pui = link[0] | |
| linklist += pui + " , " | |
| a=1 | |
| categories = page.categories | |
| categorylist = "" | |
| for category in categories.items(): | |
| pui = category[0] | |
| categorylist += pui + " , " | |
| a=1 | |
| links = page.links | |
| linklist2 = "" | |
| for link in links.items(): | |
| pui = link[0] | |
| linklist2 += pui + " , " | |
| a=1 | |
| sections = page.sections | |
| ex_dic = { | |
| 'Entity' : ["URL","Title","Summary", "Text", "Backlinks", "Links", "Categories"], | |
| 'Value': [pageurl, pagetitle, pagesummary, pagetext, linklist,linklist2, categorylist ] | |
| } | |
| df = pd.DataFrame(ex_dic) | |
| return df | |
| def save_message(name, message): | |
| now = datetime.datetime.now() | |
| timestamp = now.strftime("%Y-%m-%d %H:%M:%S") | |
| with open("chat.txt", "a") as f: | |
| f.write(f"{timestamp} - {name}: {message}\n") | |
| def press_release(): | |
| st.markdown("""🎉🎊 Breaking News! 📢📣 | |
| Introducing StreamlitWikipediaChat - the ultimate way to chat with Wikipedia and the whole world at the same time! 🌎📚👋 | |
| Are you tired of reading boring articles on Wikipedia? Do you want to have some fun while learning new things? Then StreamlitWikipediaChat is just the thing for you! 😃💻 | |
| With StreamlitWikipediaChat, you can ask Wikipedia anything you want and get instant responses! Whether you want to know the capital of Madagascar or how to make a delicious chocolate cake, Wikipedia has got you covered. 🍰🌍 | |
| But that's not all! You can also chat with other people from around the world who are using StreamlitWikipediaChat at the same time. It's like a virtual classroom where you can learn from and teach others. 🌐👨🏫👩🏫 | |
| And the best part? StreamlitWikipediaChat is super easy to use! All you have to do is type in your question and hit send. That's it! 🤯🙌 | |
| So, what are you waiting for? Join the fun and start chatting with Wikipedia and the world today! 😎🎉 | |
| StreamlitWikipediaChat - where learning meets fun! 🤓🎈""") | |
| def main(): | |
| st.title("Streamlit Chat") | |
| name = st.text_input("Enter your name") | |
| message = st.text_input("Enter a topic to share from Wikipedia") | |
| if st.button("Submit"): | |
| # wiki | |
| df = get_wiki_summaryDF(message) | |
| save_message(name, message) | |
| save_message(name, df) | |
| st.text("Message sent!") | |
| st.text("Chat history:") | |
| with open("chat.txt", "a+") as f: | |
| f.seek(0) | |
| chat_history = f.read() | |
| #st.text(chat_history) | |
| st.markdown(chat_history) | |
| countdown = st.empty() | |
| t = 60 | |
| while t: | |
| mins, secs = divmod(t, 60) | |
| countdown.text(f"Time remaining: {mins:02d}:{secs:02d}") | |
| time.sleep(1) | |
| t -= 1 | |
| if t == 0: | |
| countdown.text("Time's up!") | |
| with open("chat.txt", "a+") as f: | |
| f.seek(0) | |
| chat_history = f.read() | |
| #st.text(chat_history) | |
| st.markdown(chat_history) | |
| press_release() | |
| t = 60 | |
| if __name__ == "__main__": | |
| main() | |