Add files via upload

app.py

@@ -0,0 +1,95 @@
+#Import all neccessary libraries
+import streamlit as st
+import re
+import wikipediaapi
+import malaya
+import torch
+import tensorflow
+import pandas as pd
+from sklearn.preprocessing import OneHotEncoder, LabelEncoder
+from sklearn.model_selection import train_test_split
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.multioutput import MultiOutputClassifier
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+import psutil
+from malaya_file import *
+from knn_file import *
+
+#Page header, title
+st.set_page_config(page_title= "Malay Named Entity Recognition (NER) Model", page_icon= ":book:", layout= "wide")
+st.title(":book: Pengecaman Entiti Nama Malay (NER) model")
+st.markdown("CARA MENGGUNAKAN PROGRAM")
+st.markdown("1. Sila taip sebuah ayat atau teks tidak melebihi 500 karakter di ruangan bawah")
+st.markdown("2. Pilih model untuk melakukan proses pengecaman entiti nama (NER) berdasarkan teks")
+st.markdown("3. Klik butang 'BUAT RAMALAN' bagi memulakan program")
+st.markdown("4. Paparan bagi setiap kata serta jenis entiti akan dipaparkan pada bahagian 'HASIL RAMALAN'")
+
+#CREATE TEXT FORM
+with st.form(key= 'my_form'):
+	global kata, btn_model, df1, df2
+	kata = st.text_area(label="Sila taip teks atau ayat:", max_chars= 500)
+	
+	btn_model = st.radio("Pilih model untuk pengecaman entiti nama",
+	("KNN", "BERT", "Tiny-BERT", "ALBERT", "Tiny-ALBERT", "XLNET", "ALXLNET", "FASTFORMER", "Tiny-FASTFORMER"))
+	
+	submit_button = st.form_submit_button(label= ":arrow_right: Buat Ramalan")
+	
+	if submit_button:
+		if re.sub(r'\s+','',kata)=='':
+			st.error('Ruangan teks tidak boleh kosong.')
+			
+		elif re.match(r'\A\s*\w+\s*\Z', kata):
+			st.error("Teks atau ayat mestilah sekurang-kurangnya dua patah perkataan.")
+		
+		else:
+			if btn_model == "KNN":
+				st.write("Anda pilih model : KNN")
+				#df1 = knn_model()
+				#df2 = ramal_kata(kata)
+			else:
+				st.write("Anda pilih model transformer: ", btn_model)
+				
+			st.success("Butang hantar berfungsi!")
+
+with st.container():
+	st.write("---")
+	st.header("Hasil Ramalan")
+	st.subheader("Ayat asal")
+	st.write("##")
+	st.write(kata)
+	patah = str(len(kata.split()))
+	st.write("Bilangan perkataan : {}".format(patah))
+	st.write("##")
+	if btn_model == 'KNN':
+		df = ramal_kata(kata)
+		df_test = df.copy()
+	else:
+		df_test = malaya_model(btn_model, kata)
+	
+	entiti = sorted(df_test['entiti'].unique())
+	pilih = st.multiselect('Jenis entiti', entiti, entiti)
+	df_pilihan = df_test [ (df_test['entiti'].isin(pilih)) ]
+	st.table(df_pilihan.style.set_properties(**{'background-color': 'white', 'color': 'black'}))
+
+#About model
+with st.expander("About this app", expanded=True):
+	st.write(
+		"""     
+		-   **Pengecaman Nama Entiti Malay** adalah sebuah aplikasi pembelajaran mesin yang dibangunkan bagi mengecam entiti pada setiap token menggunakan modul MALAYA (Husein, 2018)
+		-   Projek ini adalah tugasan Final Year Project bagi Ijazah Sarjana Muda di UKM
+		-   Aplikasi ini ingin menentukan model terbaik yang boleh digunakan bagi dokumen teks subjek sejarah Bahasa Melayu
+		-   Model ini mempunyai 3 fitur utama iaitu kata asal, kata sebelum dan kata selepas. Kelas yang disasarkan ialah LOKASI, MANUSIA dan ORGANISASI
+		-   Maklumat lanjut boleh hubungi Muhd Arif Syamil bin Mohd Rahimi melalui e-mel a177313@siswa.ukm.edu.my atau 012-7049021 
+		""")
+
+process = psutil.Process(os.getpid())
+mem_size = str((process.memory_info().rss)) # in bytes, divide by 1 billion to GB
+mem_size_mb = str((process.memory_info().rss) / 1000000) 
+mem_size_gb = str((process.memory_info().rss) / 1000000000)
+st.write("Penggunaan memori: {} bytes or {} MB or {} GB".format(mem_size, mem_size_mb, mem_size_gb))
+# Dokumen Pemasyhuran Kemerdekaan 1957 telah ditulis dalam dua bahasa iaitu bahasa Melayu yang ditulis dalam Jawi dan bahasa Inggeris - No PERSON, LOCATION, ORGANISATION
+# Ketika mendarat di Lapangan Terbang Sungai Besi, tetamu kehormat telah disambut oleh Pesuruhjaya Tinggi British di Tanah Melayu, Sir Donald Charles MacGillivray dan Lady MacGillivray, Yang di-Pertuan Agong Tanah Melayu yang pertama, Tuanku Abdul Rahman diiringi Raja Permaisuri Agong dan Perdana Menteri Tanah Melayu yang pertama, Tunku Abdul Rahman.
+# Kedudukan sebuah kereta yang terjunam ke dalam Sungai Maaw di Jeti Feri Tanjung Kunyit, Sibu, semalam, sudah dikenal pasti. Jurucakap Pusat Gerakan Operasi (PGO), Jabatan Bomba dan Penyelamat Malaysia (JBPM) Sarawak, berkata kedudukan Toyota Camry di dasar sungai itu dikesan anggota Pasukan Penyelamat Di Air (PPDA) yang melakukan selaman kelima, hari ini, pada jam 3.49 petang.

knn_file.py

@@ -0,0 +1,204 @@
+#Import all neccessary libraries
+import streamlit as st
+import re
+import wikipediaapi
+import malaya
+import torch
+import tensorflow
+import pandas as pd
+from sklearn.preprocessing import OneHotEncoder, LabelEncoder
+from sklearn.model_selection import train_test_split
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.multioutput import MultiOutputClassifier
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+import psutil
+
+#LOAD PAGE AND GET TEXT
+st.cache(suppress_st_warning=True)
+def find_text():
+	global article, link, page
+	mwiki = wikipediaapi.Wikipedia(language = 'ms', extract_format = wikipediaapi.ExtractFormat.WIKI)
+	page = mwiki.page("Pemahsyuran Kemerdekaan Tanah Melayu")
+	link = page.fullurl
+	article = page.text
+	namefile = "malaytext.txt"
+	return article, page, link
+
+#CLEAN DATA
+st.cache(suppress_st_warning=True)
+def clean_data():
+	global clean_file
+	file = article
+	file1 = file.strip("\n")
+	file1 = re.sub("[=(),:;.]", "", file1)
+	file1 = file1.strip()
+	file1 = re.sub("[-']", " ", file1)
+	file1 = file1.strip()
+	file1 = file1.replace("\n", " ")
+	clean_file = file1
+	return clean_file
+
+#USE MALAYA MODULE
+st.cache(allow_output_mutation=True)
+def use_malaya():
+	global malay_pred
+	q_model = malaya.entity.transformer(model1 = 'bert', quantized = True)
+	malay_pred = q_model.predict(clean_file)
+	return malay_pred
+
+#ORGANISE DATAFRAME MODEL (NO ST.COLUMNS)
+st.cache(allow_output_mutation=True)
+def data_model():
+	global df4 #Start as LABELENCODER
+	df = pd.DataFrame(malay_pred)
+	df.columns = ['kata', 'entiti'] #1, #2
+	df['kata'].astype('str') #KIV
+	df['entiti'].astype('str')
+	df['nombor'] = df.reset_index().index #3
+	df = df.reindex(['nombor', 'kata', 'entiti'], axis = 1)
+	
+	#shift(1) moves backward by 1
+	df['SEBELUM'] = df['kata'].shift(1) #4
+	#shift(-1) moves forward by 1
+	df['SELEPAS'] = df['kata'].shift(-1) #5
+	df['TAGSEBELUM'] = df['entiti'].shift(1) #6
+	df['TAGSELEPAS'] = df['entiti'].shift(-1) #7
+	df.fillna("null", inplace=True)
+
+	#Observe entity LAIN-LAIN if it is a nuisance or otherwise
+	df1 = df.copy()
+	df1.replace("time", "OTHER", inplace=True)
+	df1.replace("event", "OTHER", inplace=True)
+	df1.replace("law", "OTHER", inplace=True)
+	df1.replace("quantity", "OTHER", inplace=True)
+	df1.replace("location", "lokasi", inplace=True)
+	df1.replace("organization", "organisasi", inplace=True)
+	df1.replace("person", "manusia", inplace=True)
+	df1.replace("OTHER", "LAIN-LAIN", inplace=True)
+
+	#ONE HOT ENCODER for LOKASI, MANUSIA dan ORGANISASI
+	ohe = OneHotEncoder()
+	ohe_entity = ohe.fit_transform(df1[['entiti']]).toarray() #8, 9, 10, 11 Expected 4 entity type
+	ohe_entity1 = pd.DataFrame(ohe_entity)
+	df2 = df1.join(ohe_entity1)
+	df2.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI']
+
+	#LABEL ENCODER for 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 
+	le = LabelEncoder()
+	le_word = le.fit_transform(df1['kata'])
+	le_word1 = pd.DataFrame(le_word)
+	df3 = df2.join(le_word1) #COLUMNS OVERLAPPED
+	df3.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS','LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA']
+	le_before = le.fit_transform(df1['SEBELUM'])
+	le_before1 = pd.DataFrame(le_before)
+	df3 = df3.join(le_before1)
+	df3.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA', 'LSEBELUM']
+	le_after = le.fit_transform(df1['SELEPAS'])
+	le_after1 = pd.DataFrame(le_after)
+	df4 = df3.join(le_after1)
+	df4.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA', 'LSEBELUM', 'LSELEPAS']
+	le_entity = le.fit_transform(df1['entiti'])
+	le_entity1 = pd.DataFrame(le_entity)
+	df4 = df4.join(le_entity1)
+	df4.columns = ['nombor', 'kata', 'entiti', 'SEBELUM', 'SELEPAS', 'TAGSEBELUM', 'TAGSELEPAS', 'LAIN-LAIN', 'LOKASI', 'MANUSIA', 'ORGANISASI', 'LKATA', 'LSEBELUM', 'LSELEPAS', 'LENTITI']
+	
+	df4['LKATA'] = df4['LKATA'].astype(str)
+	df4['LSEBELUM'] = df4['LSEBELUM'].astype(str)
+	df4['LSELEPAS'] = df4['LSELEPAS'].astype(str)
+	df4['LAIN-LAIN'] = df4['LAIN-LAIN'].astype(int)
+	df4['LOKASI'] = df4['LOKASI'].astype(int)
+	df4['ORGANISASI'] = df4['ORGANISASI'].astype(int)
+	df4['MANUSIA'] = df4['MANUSIA'].astype(int)
+	return df4
+
+#TRAIN MODEL USING KNN, MULTIOUTPUTCLASSIFIER
+st.cache(allow_output_mutation=True)
+def train_model():
+	global x, y, y_test, y_pred, knn, classifier, model_score
+	x = df4.iloc[:, [11,12,13]]
+	y = df4.iloc[:,[8,9,10]]
+	x_train, x_test, y_train, y_test = train_test_split(x, y, test_size= 0.2, random_state = 42, stratify = y)
+	knn = KNeighborsClassifier(n_neighbors= 3) #default 1st time k = 3, but entity type = 4
+	knn.fit(x_train, y_train)
+	classifier = MultiOutputClassifier(knn, n_jobs = -1)
+	classifier.fit(x_train, y_train)
+	#datax_test = x_test.values
+	datay_test = y_test.values
+	y_pred = classifier.predict(x_test)
+	model_score = classifier.score(datay_test, y_pred)
+	return x, y, y_test, y_pred, classifier, model_score
+
+#EVALUATE MODEL
+st.cache(allow_output_mutation=True)
+def evaluate_model():
+	global cm, cr, accuracy
+	y_test1 = y_test.to_numpy().flatten()
+	y_pred1 = y_pred.flatten()
+	cm = confusion_matrix(y_test1, y_pred1)
+	cr = classification_report(y_test1, y_pred1)
+	accuracy = accuracy_score(y_test1, y_pred1)
+	return cm, cr, accuracy
+
+#LOAD MODEL
+st.cache(allow_output_mutation=True)
+def knn_model():
+	result1 = find_text()
+	result2 = clean_data()
+	result3 = use_malaya()
+	result4 = data_model()
+	result5 = train_model()
+	result6 = evaluate_model()
+	return result1, result2, result3, result4, result5, result6
+
+#PREDICT WORD OUTSIDE DATA
+st.cache(allow_output_mutation=True)
+def ramal_kata(kata):
+	string = re.sub("[=(),:;.]", "", kata)
+	string1 = string.split(" ")
+	string2 = pd.DataFrame(string1, columns = ["LKATA"])
+	string2['LSEBELUM'] = string2['LKATA'].shift(1)
+	string2['LSELEPAS'] = string2['LKATA'].shift(-1)
+	string2.fillna("null", inplace=True)
+	#string1
+	#st.table(string1[:10])
+	lbl = LabelEncoder()
+	lbl_sen = lbl.fit_transform(string2['LKATA'])
+	lbl_bef = lbl.fit_transform(string2['LSEBELUM'])
+	lbl_aft = lbl.fit_transform(string2['LSELEPAS'])
+	string2 = pd.DataFrame({'LKATA':lbl_sen, 'LSEBELUM': lbl_bef, 'LSELEPAS' : lbl_aft})
+	#st.dataframe(string2.head())
+	
+	#Train, test model
+	pred_outdata = knn_model()
+	x_train, x_test, y_train, y_test = train_test_split(x, y, test_size= 0.2, random_state = 42, stratify = y)
+	pred_knn = KNeighborsClassifier(n_neighbors= 3)
+	#"classifier" VARIABLE from "TEST MODEL USING TESTING DATA"
+	kelas = MultiOutputClassifier(pred_knn, n_jobs = -1)
+	kelas.fit(x_train, y_train)
+	hasil = kelas.predict(string2)
+	#st.write(hasil)
+	
+	fin = []
+	for z in hasil:
+		if (z == [1, 0, 0]).all():
+			fin.append("LOKASI")
+		elif (z == [0, 1, 0]).all():
+			fin.append("MANUSIA")
+		elif (z == [0, 0, 1]).all():
+			fin.append("ORGANISASI")
+		else:
+			fin.append("LAIN-LAIN")
+			
+	#st.write(fin)
+	global perkata, output
+	perkata = [(key, value) for i, (key, value) in enumerate(zip(string1, fin))]
+	output = pd.DataFrame({"kata" : string1, "entiti" : fin})
+	#st.dataframe(output.transpose())
+	return output
+
+def get_data():
+	ts = output
+	return ts

malaya_file.py

@@ -0,0 +1,24 @@
+#Import all neccessary libraries
+import streamlit as st
+import re
+import wikipediaapi
+import malaya
+import torch
+import tensorflow
+import pandas as pd
+from sklearn.preprocessing import OneHotEncoder, LabelEncoder
+from sklearn.model_selection import train_test_split
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.multioutput import MultiOutputClassifier
+from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
+import numpy as np
+import matplotlib.pyplot as plt
+import tracemalloc
+
+st.cache(allow_output_mutation=True)
+def malaya_model(model_name, kata):
+	global df_malaya
+	q_model = malaya.entity.transformer(model = model_name, quantized = True)
+	malay_pred = q_model.predict(kata)
+	df_malaya = pd.DataFrame(malay_pred, columns = ['kata', 'entiti'])
+	return df_malaya

requirements.txt

@@ -0,0 +1,20 @@
+malaya<=5.0
+matplotlib<=3.6.3
+memory_profiler<=0.61.0
+numpy<=1.24.1
+pandas<=1.5.2
+psutil<=5.9.4
+scikit_learn<=1.2.0
+streamlit<=1.17.0
+tensorflow<=2.11.0
+tensorboard<=2.11.2
+tensorboard-data-server<=0.6.1
+tensorboard-plugin-wit<=1.8.1
+tensorflow-addons<=0.19.0
+tensorflow-cpu<=2.11.0
+tensorflow-estimator<=2.11.0
+tensorflow-io-gcs-filesystem<=0.29.0
+torch<=1.13.1
+torchaudio<=0.13.1
+torchvision<=0.14.1
+Wikipedia_API<=0.5.8