codeScripts/utils.py

import json
import numpy as np
import hunspell
import nltk
import nltk.corpus 
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from nltk import ne_chunk
import re
import yake
import spacy
import os
import shutil 
#dic = hunspell.Hunspell('/Users/miguel.r/Desktop/UNIR/PLenTaS/CORPUS/dict_es_ES/es_ES', '/Users/miguel.r/Desktop/es_ES/es_ES.dic')

nlp = spacy.load('es_core_news_sm') # Paquete spaCy en español (es)
nltk.download('punkt')

# Clase creada para contar sílabas de una palabra (Source: https://github.com/amunozf/separasilabas/blob/master/separasilabas.py)

#class char():
    #def __init__(self):
       # pass
    
class char_line():
    def __init__(self, word):
        self.word = word
        self.char_line = [(char, self.char_type(char)) for char in word]
        self.type_line = ''.join(chartype for char, chartype in self.char_line)
        
    def char_type(self, char):
        if char in set(['a', 'á', 'e', 'é','o', 'ó', 'í', 'ú']):
            return 'V' #strong vowel
        if char in set(['i', 'u', 'ü']):
            return 'v' #week vowel
        if char=='x':
            return 'x'
        if char=='s':
            return 's'
        else:
            return 'c'
            
    def find(self, finder):
        return self.type_line.find(finder)
        
    def split(self, pos, where):
        return char_line(self.word[0:pos+where]), char_line(self.word[pos+where:])
    
    def split_by(self, finder, where):
        split_point = self.find(finder)
        if split_point!=-1:
            chl1, chl2 = self.split(split_point, where)
            return chl1, chl2
        return self, False
     
    def __str__(self):
        return self.word
    
    def __repr__(self):
        return repr(self.word)

class silabizer():
    def __init__(self):
        self.grammar = []
        
    def split(self, chars):
        rules  = [('VV',1), ('cccc',2), ('xcc',1), ('ccx',2), ('csc',2), ('xc',1), ('cc',1), ('vcc',2), ('Vcc',2), ('sc',1), ('cs',1),('Vc',1), ('vc',1), ('Vs',1), ('vs',1)]
        for split_rule, where in rules:
            first, second = chars.split_by(split_rule,where)
            if second:
                if first.type_line in set(['c','s','x','cs']) or second.type_line in set(['c','s','x','cs']):
                    #print 'skip1', first.word, second.word, split_rule, chars.type_line
                    continue
                if first.type_line[-1]=='c' and second.word[0] in set(['l','r']):
                    continue
                if first.word[-1]=='l' and second.word[-1]=='l':
                    continue
                if first.word[-1]=='r' and second.word[-1]=='r':
                    continue
                if first.word[-1]=='c' and second.word[-1]=='h':
                    continue
                return self.split(first)+self.split(second)
        return [chars]
        
    def __call__(self, word):
        return self.split(char_line(word))

# Contador número de frases y palabras empleadas en la respuesta
def check_senteces_words(student_answer):
    
    # Tokenizing into sentences
    sentences=[]
    words=[]
    letter_per_word=[]
    syll=0 # syllables counter
    
    TokenizeAnswer = sent_tokenize(student_answer)
    for token in TokenizeAnswer:
        regex = '\\.'
        token = re.sub(regex , '', token)
        sentences.append(token)
    for i in range(len(sentences)):
        word = sentences[i].split(' ') 
        for j in range(len(word)):
            words.append(word[j])
            syllables = silabizer()
            syll=syll+len(syllables(word[j]))
            letter_per_word.append(len(word[j]))

    sentencesLenght = len(sentences)
    wordsLenght = (len(words))
    #print(f'Number of senteces used in the answer: {sentencesLenght}')
    #print(f'Number of words used in the answer: {wordsLenght}')
    
    return sentencesLenght, wordsLenght, syll, letter_per_word

# Contador faltas de ortografía
def spelling_corrector(student_answer, hunspell_aff = '/Users/javier.sanz/OneDrive - UNIR/Desktop/PLeNTas_V3/es_ES/es_ES' , hunspell_dic = '/Users/javier.sanz/OneDrive - UNIR/Desktop/PLeNTas_V3/es_ES/es_ES.dic' ):

    dic = hunspell.Hunspell(hunspell_aff, hunspell_dic)
    errors=0
    words = student_answer.split(' ')
    wrong_words = []
    for word in words:
        for element in clean_words(word):            
            if not dic.spell(element):
                #print(f'Spelling mistake: {element}')
                wrong_words.append(element)
                errors+=1        
    #print(f'Spelling mistakes: {errors}')
    return errors,wrong_words
        
# Legibilidad de la respuesta en función del índice Fernández-Huerta
def FHuertas_index(sentencesLenght, wordsLenght, syll):    
    FH = 206.84 - 0.60*(syll*100/wordsLenght) - 1.02*(sentencesLenght*100/wordsLenght) 
    FH = round(FH, 3)
    legibilidad_fh = ""
    #print(f'\nFernández-Huerta Index: {FH}')
    if 0 < FH <= 30:
        #print('Legibilidad FH: muy difícil.')
        legibilidad_fh = 'muy díficil'
    if 30 < FH <= 50:
        #print('Legibilidad FH: difícil.')  
        legibilidad_fh = 'díficil'
    if 50 < FH <= 60:
        #print('Legibilidad FH: ligeramente difícil.')
        legibilidad_fh = 'ligeramente díficil'
    if 60 < FH <= 70:
        #print('Legibilidad FH: adecuado.')
        legibilidad_fh = 'adecuado'
    if 70 < FH <= 80:
        #print('Legibilidad FH: ligeramente fácil.')
        legibilidad_fh = 'ligeramente fácil'
    if 80 < FH <= 90:
        #print('Legibilidad FH: fácil.')
        legibilidad_fh = 'fácil'
    if 90 < FH <= 100:
        #print('Legibilidad FH: muy fácil.')
        legibilidad_fh = 'muy fácil'
        
    return FH, legibilidad_fh
    
# Legibilidad de la respuesta en función del índice mu
def mu_index(sentencesLenght, wordsLenght, letter_per_word):
    med = np.mean(letter_per_word)
    var = np.var(letter_per_word)
    try:
        mu=(wordsLenght/(wordsLenght-1))*(med/var)*100
    except Exception as ex:
        print("Error on mu_index: " + str(ex))
        mu = 0

    mu=round(mu, 3)

    legibilidad_mu = ""
    #print(f'\nMu index: {mu}')
    if 0 < mu <= 30:
        #print('Legibilidad Mu: muy difícil.')
        legibilidad_mu = 'muy difícil'
    if 30 < mu <= 50:
        #print('Legibilidad Mu: difícil.')  
        legibilidad_mu = 'difícil'
    if 50 < mu <= 60:
        #print('Legibilidad Mu: ligeramente difícil.')
        legibilidad_mu = 'ligeramente difícil'
    if 60 < mu <= 70:
        #print('Legibilidad Mu: adecuado.')
        legibilidad_mu = 'adecuado'
    if 70 < mu <= 80:
        #print('Legibilidad Mu: ligeramente fácil.')
        legibilidad_mu = 'ligeramente fácil'
    if 80 < mu <= 90:
        #print('Legibilidad Mu: fácil.')
        legibilidad_mu = 'fácil'
    if 90 < mu <= 100:
        #print('Legibilidad Mu: muy fácil.')
        legibilidad_mu = 'muy fácil'
        
    return mu, legibilidad_mu

# Extractor de las kewords de un texto con librería yake
def keyword_extractor(text, numOfKeywords, language, max_ngram_size,deduplication_threshold = 0.9, features=None):
    test_keywords=[]
    # Deleting special characters and set text in lower case
    regex = '\\\n'
    text = re.sub(regex , ' ', text)
    text = text.lower()    
    custom_kw_extractor = yake.KeywordExtractor(lan=language, n=max_ngram_size, dedupLim=deduplication_threshold, top=numOfKeywords, features= features )
    keywords = custom_kw_extractor.extract_keywords(text)
    for kw in keywords:
        test_keywords.append(kw[0])
    return test_keywords

# categorización de palabras
def word_categorization(student_answer):    
    fileDocument=[]
    TokenizeAnswer = sent_tokenize(student_answer)
    for token in TokenizeAnswer:
        fileDocument.append(token)
    sentencesLenght = len(fileDocument)
    sentence=0
    while sentence < sentencesLenght:      
        # Word Tokenize sentence and Tagging the grammer tag to words (verb, noun, adj, etc...)
        word_tokens = word_tokenize(fileDocument[sentence])
        doc = nlp(fileDocument[sentence])
        pre_chunk = [(w.text, w.pos_) for w in doc]
        #print(pre_chunk)
        sentence += 1
        #pre_chunk = nltk.pos_tag(word_tokens)
        tree = ne_chunk(pre_chunk) # same tagging than before
        #grammer_np = ("NP: {<DT>?<JJ>*<NN>}")
        
        # Chunking rules to filter out:
        grammer_np = ("NP: {<DET>?<ADJ>*<NOUN>*<VERB>}")
        grammar = r"""

          NP: {<DT|PP\$>?<JJ>*<NN>}   # chunk determiner/possessive, adjectives and nouns

              {<NNP>+}                # chunk sequences of proper nouns

        """
        chunk_parser = nltk.RegexpParser(grammer_np)
        chunk_result = chunk_parser.parse(tree)

#..................................................................................................
def char_split(word, character):
    palabra1=""
    palabra2=""
    found = 0
    for w in word:
        if w == character and not found:
            found = 1
        else:
            if not found:
              palabra1 = palabra1 + w
            else:
              palabra2 = palabra2 + w

    return [palabra1, palabra2]

def clean_words(string):
    words_sentence = []
    for w in string:
      if not w.isalnum():
        if char_split(string, w)[0] != "":
            words_sentence.append(char_split(string, w)[0])
        string = char_split(string, w)[len(char_split(string, w))-1]

    if string != "":
        words_sentence.append(string)
    return words_sentence

def getNameFile(string):
    directories = string.split("/")
    return re.sub(".json","", directories[len(directories)-1])


def getIDrange(rango_ID, df):
    if rango_ID == "All":
        IDs = list(range(len(df['hashed_id'])))
    else:
        rango = []
        r= rango_ID.split(",")
        for i in r:
            c_w= clean_words(i)
            if len(c_w) == 2:
                rango= rango + list(range(int(c_w[0]) -1 ,int(c_w[1])))
            elif len(c_w) == 1:
                rango.append(int(c_w[0]) -1)
        IDs = rango

    return IDs

def save_json(path, data, isIndent = True):
    if isIndent:
        json_object = json.dumps(data, indent = 11, ensure_ascii= False)
    else:
        json_object = json.dumps(data, ensure_ascii= False)
    # Writing output to a json file
    with open(path, "w") as outfile:
        outfile.write(json_object)


def load_json(path):
    with open(path, "r", encoding="utf8") as f:
        data = json.loads("[" + f.read().replace("}\n{", "},\n{") + "]")
    
    return data

def load_json_dtset(path):
    with open(path, "r", encoding="latin-1") as f:
        data = json.loads("[" + f.read().replace("}\n{", "},\n{") + "]")
    
    return data

    
def splitResponse(respuesta_alumno_raw):
    #pre-processing the student's response
    regex = '\\\n'
    respuesta_alumno = re.sub(regex , ' ', respuesta_alumno_raw)
    respuesta_alumno = respuesta_alumno.lower()

    #stacking each sentence of the student's response
    sentences=[]                        
    TokenizeAnswer = sent_tokenize(respuesta_alumno)
    for token in TokenizeAnswer:
        regex = '\\.'
        token = re.sub(regex , '', token)
        sentences.append(token)

    return sentences
        
def create_file_path(file, doctype):
    """

    This function is to create relative paths to store data.

    Inputs:

        file: the file or subpath + file where the info is to be stored

        doctype: 1- Info from the api, 2- Output documents, 3- Images, 4- Bert models/documents

    Outputs:

        path: the generated path

    """
    if doctype == 1:
        path = "api/" + file
    elif doctype == 2:
        path = "archivos/OutputFiles2/" + file
    elif doctype == 3:
        path = "archivos/Images/" + file
    else:
        path = "codeScripts/Dependencies/BERT-models/Prueba3/" + file
    return path

def remove(path):
    """ param <path> could either be relative or absolute. """
    if os.path.isfile(path) or os.path.islink(path):
        os.remove(path)  # remove the file
    elif os.path.isdir(path):
        shutil.rmtree(path)  # remove dir and all contains
    else:
        raise ValueError("file {} is not a file or dir.".format(path))