knn_file.py

#Import all neccessary libraries
import streamlit as st
import re
import wikipediaapi
import malaya
import torch
import tensorflow
import pandas as pd
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
import numpy as np
import matplotlib.pyplot as plt
import os
import psutil

#LOAD PAGE AND GET TEXT
st.cache(suppress_st_warning=True)
def find_text():
	global article, link, page
	mwiki = wikipediaapi.Wikipedia(language = 'ms', extract_format = wikipediaapi.ExtractFormat.WIKI)
	page = mwiki.page("Pemahsyuran Kemerdekaan Tanah Melayu")
	link = page.fullurl
	article = page.text
	namefile = "malaytext.txt"
	return article, page, link

#CLEAN DATA
st.cache(suppress_st_warning=True)
def clean_data():
	global clean_file
	file = article
	file1 = file.strip("\n")
	file1 = re.sub("[=(),:;.]", "", file1)
	file1 = file1.strip()
	file1 = re.sub("[-']", " ", file1)
	file1 = file1.strip()
	file1 = file1.replace("\n", " ")
	clean_file = file1
	return clean_file

#USE MALAYA MODULE
st.cache(allow_output_mutation=True)
def use_malaya():
	global malay_pred
	q_model = malaya.entity.transformer(model1 = 'bert', quantized = True)
	malay_pred = q_model.predict(clean_file)
	return malay_pred

#ORGANISE DATAFRAME MODEL (NO ST.COLUMNS)
st.cache(allow_output_mutation=True)
def data_model():
	global df4 #Start as LABELENCODER
	df = pd.DataFrame(malay_pred)
	df.columns = ['kata', 'entiti'] #1, #2
	df['kata'].astype('str') #KIV
	df['entiti'].astype('str')
	df['nombor'] = df.reset_index().index #3
	df = df.reindex(['nombor', 'kata', 'entiti'], axis = 1)
	
	#shift(1) moves backward by 1
	df['SEBELUM'] = df['kata'].shift(1) #4
	#shift(-1) moves forward by 1
	df['SELEPAS'] = df['kata'].shift(-1) #5
	df['TAGSEBELUM'] = df['entiti'].shift(1) #6
	df['TAGSELEPAS'] = df['entiti'].shift(-1) #7
	df.fillna("null", inplace=True)

	#Observe entity LAIN-LAIN if it is a nuisance or otherwise
	df1 = df.copy()
	df1.replace("time", "OTHER", inplace=True)
	df1.replace("event", "OTHER", inplace=True)
	df1.replace("law", "OTHER", inplace=True)
	df1.replace("quantity", "OTHER", inplace=True)
	df1.replace("location", "lokasi", inplace=True)
	df1.replace("organization", "organisasi", inplace=True)
	df1.replace("person", "manusia", inplace=True)
	df1.replace("OTHER", "LAIN-LAIN", inplace=True)

	#ONE HOT ENCODER for LOKASI, MANUSIA dan ORGANISASI
	ohe = OneHotEncoder()
	ohe_entity = ohe.fit_transform(df1[['entiti']]).toarray() #8, 9, 10, 11 Expected 4 entity type
	ohe_entity1 = pd.DataFrame(ohe_entity)
	df2 = df1.join(ohe_entity1)
	df2.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI']

	#LABEL ENCODER for 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 
	le = LabelEncoder()
	le_word = le.fit_transform(df1['kata'])
	le_word1 = pd.DataFrame(le_word)
	df3 = df2.join(le_word1) #COLUMNS OVERLAPPED
	df3.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS','LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA']
	le_before = le.fit_transform(df1['SEBELUM'])
	le_before1 = pd.DataFrame(le_before)
	df3 = df3.join(le_before1)
	df3.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA', 'LSEBELUM']
	le_after = le.fit_transform(df1['SELEPAS'])
	le_after1 = pd.DataFrame(le_after)
	df4 = df3.join(le_after1)
	df4.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA', 'LSEBELUM', 'LSELEPAS']
	le_entity = le.fit_transform(df1['entiti'])
	le_entity1 = pd.DataFrame(le_entity)
	df4 = df4.join(le_entity1)
	df4.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA', 'LSEBELUM', 'LSELEPAS', 'LENTITI']
	
	df4['LKATA'] = df4['LKATA'].astype(str)
	df4['LSEBELUM'] = df4['LSEBELUM'].astype(str)
	df4['LSELEPAS'] = df4['LSELEPAS'].astype(str)
	df4['LAIN-LAIN'] = df4['LAIN-LAIN'].astype(int)
	df4['LOKASI'] = df4['LOKASI'].astype(int)
	df4['ORGANISASI'] = df4['ORGANISASI'].astype(int)
	df4['MANUSIA'] = df4['MANUSIA'].astype(int)
	return df4

#TRAIN MODEL USING KNN, MULTIOUTPUTCLASSIFIER
st.cache(allow_output_mutation=True)
def train_model():
	global x, y, y_test, y_pred, knn, classifier, model_score
	x = df4.iloc[:, [11,12,13]]
	y = df4.iloc[:,[8,9,10]]
	x_train, x_test, y_train, y_test = train_test_split(x, y, test_size= 0.2, random_state = 42, stratify = y)
	knn = KNeighborsClassifier(n_neighbors= 3) #default 1st time k = 3, but entity type = 4
	knn.fit(x_train, y_train)
	classifier = MultiOutputClassifier(knn, n_jobs = -1)
	classifier.fit(x_train, y_train)
	#datax_test = x_test.values
	datay_test = y_test.values
	y_pred = classifier.predict(x_test)
	model_score = classifier.score(datay_test, y_pred)
	return x, y, y_test, y_pred, classifier, model_score

#EVALUATE MODEL
st.cache(allow_output_mutation=True)
def evaluate_model():
	global cm, cr, accuracy
	y_test1 = y_test.to_numpy().flatten()
	y_pred1 = y_pred.flatten()
	cm = confusion_matrix(y_test1, y_pred1)
	cr = classification_report(y_test1, y_pred1)
	accuracy = accuracy_score(y_test1, y_pred1)
	return cm, cr, accuracy

#LOAD MODEL
st.cache(allow_output_mutation=True)
def knn_model():
	result1 = find_text()
	result2 = clean_data()
	result3 = use_malaya()
	result4 = data_model()
	result5 = train_model()
	result6 = evaluate_model()
	return result1, result2, result3, result4, result5, result6

#PREDICT WORD OUTSIDE DATA
st.cache(allow_output_mutation=True)
def ramal_kata(kata):
	string = re.sub("[=(),:;.]", "", kata)
	string1 = string.split(" ")
	string2 = pd.DataFrame(string1, columns = ["LKATA"])
	string2['LSEBELUM'] = string2['LKATA'].shift(1)
	string2['LSELEPAS'] = string2['LKATA'].shift(-1)
	string2.fillna("null", inplace=True)
	#string1
	#st.table(string1[:10])
	lbl = LabelEncoder()
	lbl_sen = lbl.fit_transform(string2['LKATA'])
	lbl_bef = lbl.fit_transform(string2['LSEBELUM'])
	lbl_aft = lbl.fit_transform(string2['LSELEPAS'])
	string2 = pd.DataFrame({'LKATA':lbl_sen, 'LSEBELUM': lbl_bef, 'LSELEPAS' : lbl_aft})
	#st.dataframe(string2.head())
	
	#Train, test model
	pred_outdata = knn_model()
	x_train, x_test, y_train, y_test = train_test_split(x, y, test_size= 0.2, random_state = 42, stratify = y)
	pred_knn = KNeighborsClassifier(n_neighbors= 3)
	#"classifier" VARIABLE from "TEST MODEL USING TESTING DATA"
	kelas = MultiOutputClassifier(pred_knn, n_jobs = -1)
	kelas.fit(x_train, y_train)
	hasil = kelas.predict(string2)
	#st.write(hasil)
	
	fin = []
	for z in hasil:
		if (z == [1, 0, 0]).all():
			fin.append("LOKASI")
		elif (z == [0, 1, 0]).all():
			fin.append("MANUSIA")
		elif (z == [0, 0, 1]).all():
			fin.append("ORGANISASI")
		else:
			fin.append("LAIN-LAIN")
			
	#st.write(fin)
	global perkata, output
	perkata = [(key, value) for i, (key, value) in enumerate(zip(string1, fin))]
	output = pd.DataFrame({"kata" : string1, "entiti" : fin})
	#st.dataframe(output.transpose())
	return output

def get_data():
	ts = output
	return ts