Spaces:
Runtime error
Runtime error
| # | |
| # Copyright (c) 2013-present, Anoop Kunchukuttan | |
| # All rights reserved. | |
| # | |
| # This source code is licensed under the MIT license found in the | |
| # LICENSE file in the root directory of this source tree. | |
| # | |
| import codecs, sys | |
| from indicnlp.script import indic_scripts as si | |
| import re | |
| chillu_char_map= { | |
| '\u0d7a': '\u0d23', | |
| '\u0d7b': '\u0d28', | |
| '\u0d7c': '\u0d30', | |
| '\u0d7d': '\u0d32', | |
| '\u0d7e': '\u0d33', | |
| '\u0d7f': '\u0d15', | |
| } | |
| char_chillu_map= {} | |
| for k,v in chillu_char_map.items(): | |
| char_chillu_map[v]=k | |
| def normalize_malayalam(word): | |
| word_mask=re.sub(r'[0-9]','0',word) | |
| # instead of chillu characters, use consonant+halant | |
| for chillu,char in chillu_char_map.items(): | |
| word=word.replace(chillu,'{}\u0d4d'.format(char)) | |
| word_mask=word_mask.replace(chillu,'41') | |
| word_mask=re.sub(r'[^0-9]','0',word_mask) | |
| return word, word_mask | |
| def denormalize_malayalam(word, word_mask): | |
| word=list(word) | |
| word_mask=list(word_mask) | |
| ## pattern 4 | |
| idx=0 | |
| while idx>=0: | |
| try: | |
| idx=word_mask.index('4',idx) | |
| word[idx:idx+2]=char_chillu_map[word[idx]] | |
| word_mask[idx:idx+2]='0' | |
| start=idx | |
| except ValueError as e: | |
| break | |
| return ''.join(word) | |
| def normalize_punjabi(word): | |
| word_mask=re.sub(r'[0-9]','0',word) | |
| ## replace tippi with anusvaar | |
| word=word.replace('\u0a70','\u0a02') | |
| word_mask=word_mask.replace('\u0a70','2') | |
| ## replace addak+consonant with consonat+halant+consonant | |
| word=re.sub(r'\u0a71(.)','\\1\u0a4d\\1',word) | |
| word_mask=re.sub(r'\u0a71(.)','311',word_mask) | |
| word_mask=re.sub(r'[^0-9]','0',word_mask) | |
| return word, word_mask | |
| def denormalize_punjabi(word, word_mask): | |
| word=list(word) | |
| word_mask=list(word_mask) | |
| ## pattern 2 | |
| idx=0 | |
| while idx>=0: | |
| try: | |
| idx=word_mask.index('2',idx) | |
| word[idx]='\u0a70' | |
| word_mask[idx]='0' | |
| start=idx | |
| except ValueError as e: | |
| break | |
| ## pattern 3 | |
| idx=0 | |
| while idx>=0: | |
| try: | |
| idx=word_mask.index('3',idx) | |
| word[idx:idx+3]='\u0a71{}'.format(word[idx]) | |
| word_mask[idx:idx+3]='00' | |
| start=idx | |
| except ValueError as e: | |
| break | |
| return ''.join(word) | |
| def char_backoff(syllables_list,vocab): | |
| syllables_final=[] | |
| if vocab is None: | |
| syllables_final=syllables_list | |
| else: | |
| for s in syllables_list: | |
| if s in vocab: | |
| syllables_final.append(s) | |
| else: | |
| for x in s: | |
| syllables_final.append(x) | |
| return syllables_final | |
| def orthographic_syllabify_improved(word,lang,vocab=None): | |
| word_mask=['0']*len(word) | |
| if lang=='ml': | |
| word, word_mask = normalize_malayalam(word) | |
| word=word | |
| elif lang=='pa': | |
| word, word_mask = normalize_punjabi(word) | |
| p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word] | |
| syllables=[] | |
| syllables_mask=[] | |
| for i in range(len(word)): | |
| v=p_vectors[i] | |
| syllables.append(word[i]) | |
| syllables_mask.append(word_mask[i]) | |
| ### simplified syllabification | |
| #if i+1<len(word) and \ | |
| # (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])): | |
| # syllables.append(u' ') | |
| # syllables_mask.append(u'0') | |
| #elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v): | |
| # syllables.append(u' ') | |
| # syllables_mask.append(u'0') | |
| #elif i+1<len(word) and \ | |
| # (si.is_consonant(v) or si.is_nukta(v)) and \ | |
| # (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])): | |
| # syllables.append(u' ') | |
| # syllables_mask.append(u'0') | |
| #### better syllabification | |
| if i+1<len(word) and (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])): | |
| syllables.append(' ') | |
| syllables_mask.append('0') | |
| elif not si.is_valid(v) or si.is_misc(v) : | |
| syllables.append(' ') | |
| syllables_mask.append('0') | |
| elif si.is_vowel(v): | |
| anu_nonplos= ( i+2<len(word) and \ | |
| si.is_anusvaar(p_vectors[i+1]) and \ | |
| not si.is_plosive(p_vectors[i+2])\ | |
| ) | |
| anu_eow= ( i+2==len(word) and \ | |
| si.is_anusvaar(p_vectors[i+1]) ) | |
| if not(anu_nonplos or anu_eow): | |
| syllables.append(' ') | |
| syllables_mask.append('0') | |
| elif i+1<len(word) and \ | |
| (si.is_consonant(v) or si.is_nukta(v)): | |
| if si.is_consonant(p_vectors[i+1]): | |
| syllables.append(' ') | |
| syllables_mask.append('0') | |
| elif si.is_vowel(p_vectors[i+1]) and \ | |
| not si.is_dependent_vowel(p_vectors[i+1]): | |
| syllables.append(' ') | |
| syllables_mask.append('0') | |
| elif si.is_anusvaar(p_vectors[i+1]): | |
| anu_nonplos= ( i+2<len(word) and \ | |
| not si.is_plosive(p_vectors[i+2])\ | |
| ) | |
| anu_eow= i+2==len(word) | |
| if not(anu_nonplos or anu_eow): | |
| syllables.append(' ') | |
| syllables_mask.append('0') | |
| syllables_mask=''.join(syllables_mask) | |
| syllables=''.join(syllables) | |
| #assert len(syllables_mask) == len(syllables) | |
| #assert syllables_mask.find('01') == -1 | |
| if syllables_mask.find('01') >= 0: | |
| print('Warning') | |
| if lang=='ml': | |
| syllables = denormalize_malayalam(syllables,syllables_mask) | |
| elif lang=='pa': | |
| syllables = denormalize_punjabi(syllables,syllables_mask) | |
| syllables_list = syllables.strip().split(' ') | |
| return(char_backoff(syllables_list,vocab)) | |
| def orthographic_syllabify(word,lang,vocab=None): | |
| p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word] | |
| syllables=[] | |
| for i in range(len(word)): | |
| v=p_vectors[i] | |
| syllables.append(word[i]) | |
| ### simplified syllabification | |
| #if i+1<len(word) and \ | |
| # (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])): | |
| # syllables.append(u' ') | |
| #elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v): | |
| # syllables.append(u' ') | |
| #elif i+1<len(word) and \ | |
| # (si.is_consonant(v) or si.is_nukta(v)) and \ | |
| # (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])): | |
| # syllables.append(u' ') | |
| #### better syllabification | |
| if i+1<len(word) and (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])): | |
| syllables.append(' ') | |
| elif not si.is_valid(v) or si.is_misc(v) : | |
| syllables.append(' ') | |
| elif si.is_vowel(v): | |
| anu_nonplos= ( i+2<len(word) and \ | |
| si.is_anusvaar(p_vectors[i+1]) and \ | |
| not si.is_plosive(p_vectors[i+2])\ | |
| ) | |
| anu_eow= ( i+2==len(word) and \ | |
| si.is_anusvaar(p_vectors[i+1]) ) | |
| if not(anu_nonplos or anu_eow): | |
| syllables.append(' ') | |
| elif i+1<len(word) and \ | |
| (si.is_consonant(v) or si.is_nukta(v)): | |
| if si.is_consonant(p_vectors[i+1]): | |
| syllables.append(' ') | |
| elif si.is_vowel(p_vectors[i+1]) and \ | |
| not si.is_dependent_vowel(p_vectors[i+1]): | |
| syllables.append(' ') | |
| elif si.is_anusvaar(p_vectors[i+1]): | |
| anu_nonplos= ( i+2<len(word) and \ | |
| not si.is_plosive(p_vectors[i+2])\ | |
| ) | |
| anu_eow= i+2==len(word) | |
| if not(anu_nonplos or anu_eow): | |
| syllables.append(' ') | |
| syllables_list = ''.join(syllables).strip().split(' ') | |
| return(char_backoff(syllables_list,vocab)) | |
| def orthographic_simple_syllabify(word,lang,vocab=None): | |
| p_vectors=[si.get_phonetic_feature_vector(c,lang) for c in word] | |
| syllables=[] | |
| for i in range(len(word)): | |
| v=p_vectors[i] | |
| syllables.append(word[i]) | |
| ## simplified syllabification | |
| if i+1<len(word) and \ | |
| (not si.is_valid(p_vectors[i+1]) or si.is_misc(p_vectors[i+1])): | |
| syllables.append(' ') | |
| elif not si.is_valid(v) or si.is_misc(v) or si.is_vowel(v): | |
| syllables.append(' ') | |
| elif i+1<len(word) and \ | |
| (si.is_consonant(v) or si.is_nukta(v)) and \ | |
| (si.is_consonant(p_vectors[i+1]) or si.is_anusvaar(p_vectors[i+1])): | |
| syllables.append(' ') | |
| syllables_list = ''.join(syllables).strip().split(' ') | |
| return(char_backoff(syllables_list,vocab)) | |