Spaces:
Runtime error
Runtime error
File size: 4,881 Bytes
211220e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 |
import nltk
nltk.download('popular')
nltk.download('punkt_tab')
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
import pickle
import numpy as np
from keras.models import load_model
model = load_model('model.h5')
import json
import random
from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
import spacy
from spacy.language import Language
from spacy_langdetect import LanguageDetector
# translator pipeline for english to swahili translations
eng_swa_tokenizer = AutoTokenizer.from_pretrained("Rogendo/en-sw")
eng_swa_model = AutoModelForSeq2SeqLM.from_pretrained("Rogendo/en-sw")
eng_swa_translator = pipeline(
"text2text-generation",
model = eng_swa_model,
tokenizer = eng_swa_tokenizer,
)
def translate_text_eng_swa(text):
translated_text = eng_swa_translator(text, max_length=128, num_beams=5)[0]['generated_text']
return translated_text
# translator pipeline for swahili to english translations
swa_eng_tokenizer = AutoTokenizer.from_pretrained("Rogendo/sw-en")
swa_eng_model = AutoModelForSeq2SeqLM.from_pretrained("Rogendo/sw-en")
swa_eng_translator = pipeline(
"text2text-generation",
model = swa_eng_model,
tokenizer = swa_eng_tokenizer,
)
def translate_text_swa_eng(text):
translated_text = swa_eng_translator(text,max_length=128, num_beams=5)[0]['generated_text']
return translated_text
def get_lang_detector(nlp, name):
return LanguageDetector()
nlp = spacy.load("en_core_web_sm")
Language.factory("language_detector", func=get_lang_detector)
nlp.add_pipe('language_detector', last=True)
intents = json.loads(open('intents.json').read())
words = pickle.load(open('texts.pkl','rb'))
classes = pickle.load(open('labels.pkl','rb'))
def clean_up_sentence(sentence):
sentence_words = nltk.word_tokenize(sentence)
sentence_words = [lemmatizer.lemmatize(word.lower()) for word in sentence_words]
return sentence_words
def bow(sentence, words, show_details=True):
sentence_words = clean_up_sentence(sentence)
bag = [0]*len(words)
for s in sentence_words:
for i,w in enumerate(words):
if w == s:
bag[i] = 1
if show_details:
print ("found in bag: %s" % w)
return(np.array(bag))
def predict_class(sentence, model):
p = bow(sentence, words,show_details=False)
res = model.predict(np.array([p]))[0]
ERROR_THRESHOLD = 0.25
results = [[i,r] for i,r in enumerate(res) if r>ERROR_THRESHOLD]
results.sort(key=lambda x: x[1], reverse=True)
return_list = []
for r in results:
return_list.append({"intent": classes[r[0]], "probability": str(r[1])})
return return_list
def getResponse(ints, intents_json):
if ints:
tag = ints[0]['intent']
list_of_intents = intents_json['intents']
for i in list_of_intents:
if i['tag'] == tag:
result = random.choice(i['responses'])
break
return result
else:
return "Sorry, I didn't understand that."
def chatbot_response(msg):
doc = nlp(msg)
detected_language = doc._.language['language']
print(f"Detected language chatbot_response:- {detected_language}")
chatbotResponse = "Loading bot response..........."
if detected_language == "en":
res = getResponse(predict_class(msg, model), intents)
chatbotResponse = res
print("en_sw chatbot_response:- ", res)
elif detected_language == 'sw':
translated_msg = translate_text_swa_eng(msg)
res = getResponse(predict_class(translated_msg, model), intents)
chatbotResponse = translate_text_eng_swa(res)
print("sw_en chatbot_response:- ", chatbotResponse)
return chatbotResponse
from flask import Flask, render_template, request
app = Flask(__name__)
app.static_folder = 'static'
@app.route("/")
def home():
return render_template("index.html")
@app.route("/get")
def get_bot_response():
userText = request.args.get('msg')
print("get_bot_response:- " + userText)
doc = nlp(userText)
detected_language = doc._.language['language']
print(f"Detected language get_bot_response:- {detected_language}")
bot_response_translate = "Loading bot response..........."
if detected_language == "en":
bot_response_translate = userText
print("en_sw get_bot_response:-", bot_response_translate)
elif detected_language == 'sw':
bot_response_translate = translate_text_swa_eng(userText)
print("sw_en get_bot_response:-", bot_response_translate)
chatbot_response_text = chatbot_response(bot_response_translate)
if detected_language == 'sw':
chatbot_response_text = translate_text_eng_swa(chatbot_response_text)
return chatbot_response_text
if __name__ == "__main__":
app.run()
|