from transformers import FSMTForConditionalGeneration, FSMTTokenizer from transformers import AutoModelForSequenceClassification from transformers import AutoTokenizer from langdetect import detect from newspaper import Article from PIL import Image import streamlit as st import requests import torch st.markdown("## Prediction of Fakeness by Given URL") background = Image.open('logo.jpg') st.image(background) st.markdown(f"### Article URL") text = st.text_area("Insert some url here", value="https://en.globes.co.il/en/article-yandex-looks-to-expand-activities-in-israel-1001406519") @st.cache(allow_output_mutation=True) def get_models_and_tokenizers(): model_name = 'distilbert-base-uncased-finetuned-sst-2-english' model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2) model.eval() tokenizer = AutoTokenizer.from_pretrained(model_name) model.load_state_dict(torch.load('model.pth')) model_name_translator = "facebook/wmt19-ru-en" tokenizer_translator = FSMTTokenizer.from_pretrained(model_name_translator) model_translator = FSMTForConditionalGeneration.from_pretrained(model_name_translator) model_translator.eval() return model, tokenizer, model_translator, tokenizer_translator model, tokenizer, model_translator, tokenizer_translator = get_models_and_tokenizers() article = Article(text) article.download() article.parse() concated_text = article.title + '. ' + article.text lang = detect(concated_text) st.markdown(f"### Language detection") if lang == 'ru': with st.spinner('Waiting for translation: '): st.markdown(f"The language of this article is {lang.upper()} so we translated it!") input_ids = tokenizer_translator.encode(concated_text, return_tensors="pt", max_length=512, truncation=True) outputs = model_translator.generate(input_ids) decoded = tokenizer_translator.decode(outputs[0], skip_special_tokens=True) st.markdown("### Translated Text") st.markdown(f"{decoded[:777]}") concated_text = decoded else: st.markdown(f"The language of this article for sure: {lang.upper()}!") st.markdown("### Extracted Text") st.markdown(f"{concated_text[:777]}") tokens_info = tokenizer(concated_text, truncation=True, return_tensors="pt") with torch.no_grad(): raw_predictions = model(**tokens_info) softmaxed = int(torch.nn.functional.softmax(raw_predictions.logits[0], dim=0)[1] * 100) st.markdown("### Fakeness Prediction") st.progress(softmaxed) st.markdown(f"This is fake by **{softmaxed}%**!") if (softmaxed > 70): st.error('We would not trust this text!') elif (softmaxed > 40): st.warning('We are not sure about this text!') else: st.success('We would trust this text!')