Spaces:
Runtime error
Runtime error
File size: 3,440 Bytes
7e8bf02 91b1844 7e8bf02 91b1844 7e8bf02 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
import streamlit as st
import pickle
import numpy as np
import pandas as pd
import re
import tensorflow
from tensorflow import keras
from keras.preprocessing import text,sequence,utils
import html
import string
import nltk
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
from tensorflow.keras.preprocessing.text import text_to_word_sequence
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras import models
from tensorflow.keras import layers
from tensorflow.keras import losses
from tensorflow.keras import metrics
from tensorflow.keras import optimizers
from tensorflow.keras.utils import plot_model
def remove_special_chars(text):
re1 = re.compile(r' +')
x1 = text.lower().replace('#39;', "'").replace('amp;', '&').replace('#146;', "'").replace(
'nbsp;', ' ').replace('#36;', '$').replace('\\n', "\n").replace('quot;', "'").replace(
'<br />', "\n").replace('\\"', '"').replace('<unk>', 'u_n').replace(' @.@ ', '.').replace(
' @-@ ', '-').replace('\\', ' \\ ')
return re1.sub(' ', html.unescape(x1))
def to_lowercase(text):
return text.lower()
def remove_punctuation(text):
"""Remove punctuation from list of tokenized words"""
translator = str.maketrans('', '', string.punctuation)
return text.translate(translator)
def replace_numbers(text):
"""Replace all interger occurrences in list of tokenized words with textual representation"""
return re.sub(r'\d+', '', text)
def remove_whitespaces(text):
return text.strip()
def remove_stopwords(words, stop_words):
return [word for word in words if word not in stop_words]
def stem_words(words):
"""Stem words in text"""
stemmer = PorterStemmer()
return [stemmer.stem(word) for word in words]
def lemmatize_words(words):
"""Lemmatize words in text"""
lemmatizer = WordNetLemmatizer()
return [lemmatizer.lemmatize(word) for word in words]
def lemmatize_verbs(words):
"""Lemmatize verbs in text"""
lemmatizer = WordNetLemmatizer()
return ' '.join([lemmatizer.lemmatize(word, pos='v') for word in words])
def text2words(text):
return word_tokenize(text)
def clean_text( text):
text = remove_special_chars(text)
text = remove_punctuation(text)
text = to_lowercase(text)
text = replace_numbers(text)
words = text2words(text)
words = remove_stopwords(words, stop_words)
#words = stem_words(words)# Either stem ovocar lemmatize
words = lemmatize_words(words)
words = lemmatize_verbs(words)
return ''.join(words)
df = pd.read_csv('train.csv.zip')
df['comment_text'] = df['comment_text'].apply(lambda x: clean_text(x))
model = pickle.load(open('tox_model.h5','rb'))
st.title('Toxic comment classification')
input = st.text_area('Enter your comment')
input = input.apply(lambda x: clean_text(x))
tok = Tokenizer(num_words=1000, oov_token='UNK')
tok.fit_on_texts(df['comment_text'] )
x_test = tok.texts_to_sequence(input)
input_text = pad_sequences(x_test,
maxlen=50,
truncating='post',
padding='post'
)
if input:
out = model.predict(input_text)
st.json(out)
|