SourceCode/Views/Tokenizer/test_Tokenizer.py

from Tokenizer import *

t = Tokenizer("بسم الله الرحمن الرحيم. الحمد لله رب العالمين.");
#t = Tokenizer("the avg: 8.3 time 2:23:21 am... This is   nice...");
print(t.FinalCharsType);
print(t);