File size: 8,770 Bytes
e50fe35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
# 
#  Copyright (c) 2013-present, Anoop Kunchukuttan
#  All rights reserved.
#  
#  This source code is licensed under the MIT license found in the
#  LICENSE file in the root directory of this source tree.
# 

import pandas as pd
import numpy as np
import os

from indicnlp import common
from indicnlp.common import IndicNlpException
from indicnlp import langinfo as li

###
# Phonetic Information about script characters 
###

""" Phonetic data about all languages except Tamil """
ALL_PHONETIC_DATA=None

""" Phonetic data for Tamil """
TAMIL_PHONETIC_DATA=None

""" Phonetic vector for all languages except Tamil """
ALL_PHONETIC_VECTORS=None

""" Phonetic vector for Tamil """
TAMIL_PHONETIC_VECTORS=None

""" Length of phonetic vector """
PHONETIC_VECTOR_LENGTH=38

""" Start offset for the phonetic feature vector in the phonetic data vector """
PHONETIC_VECTOR_START_OFFSET=6

## PHONETIC PROPERTIES in order in which they occur in the vector 
## This list must be in sync with the keys in the PV_PROP_RANGES dictionary 
PV_PROP=['basic_type',
    'vowel_length',
    'vowel_strength',
    'vowel_status',
    'consonant_type',
    'articulation_place',
    'aspiration',
    'voicing',
    'nasalization',
    'vowel_horizontal',
    'vowel_vertical',
    'vowel_roundness',
    ]

### 
# Bit vector ranges for various properties 
###

PV_PROP_RANGES={
        'basic_type': [0,6],
        'vowel_length': [6,8],
        'vowel_strength': [8,11],
        'vowel_status': [11,13],
        'consonant_type': [13,18],
        'articulation_place': [18,23],
        'aspiration': [23,25],
        'voicing': [25,27],
        'nasalization': [27,29],
        'vowel_horizontal': [29,32],
        'vowel_vertical': [32,36],
        'vowel_roundness': [36,38],
        }


####
# Indexes into the Phonetic Vector 
####
PVIDX_BT_VOWEL=0
PVIDX_BT_CONSONANT=1
PVIDX_BT_NUKTA=2
PVIDX_BT_HALANT=3
PVIDX_BT_ANUSVAAR=4
PVIDX_BT_MISC=5
PVIDX_BT_S=PVIDX_BT_VOWEL 
PVIDX_BT_E=PVIDX_BT_MISC+1

PVIDX_VSTAT_DEP=12

#####
# Unicode information about characters 
#####

SCRIPT_OFFSET_START=0
SCRIPT_OFFSET_RANGE=0x80

def init():
    """
    To be called by library loader, do not call it in your program 
    """

    global ALL_PHONETIC_DATA, ALL_PHONETIC_VECTORS, TAMIL_PHONETIC_DATA, TAMIL_PHONETIC_VECTORS, PHONETIC_VECTOR_LENGTH, PHONETIC_VECTOR_START_OFFSET

    ALL_PHONETIC_DATA=pd.read_csv(os.path.join(common.get_resources_path(),'script','all_script_phonetic_data.csv'),encoding='utf-8')    
    TAMIL_PHONETIC_DATA=pd.read_csv(os.path.join(common.get_resources_path(),'script','tamil_script_phonetic_data.csv'),encoding='utf-8')    

    ALL_PHONETIC_VECTORS= ALL_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values
    TAMIL_PHONETIC_VECTORS=TAMIL_PHONETIC_DATA.iloc[:,PHONETIC_VECTOR_START_OFFSET:].values

    PHONETIC_VECTOR_LENGTH=ALL_PHONETIC_VECTORS.shape[1]

def is_supported_language(lang): 
    return lang in list(li.SCRIPT_RANGES.keys())

def get_offset(c,lang): 
    if not is_supported_language(lang): 
        raise IndicNlpException('Language {}  not supported'.format(lang))
    return ord(c)-li.SCRIPT_RANGES[lang][0]

def offset_to_char(off,lang): 
    """
    Applicable to Brahmi derived Indic scripts 
    """
    if not is_supported_language(lang): 
        raise IndicNlpException('Language {}  not supported'.format(lang))
    return chr(off+li.SCRIPT_RANGES[lang][0])

def is_indiclang_char(c,lang): 
    """
    Applicable to Brahmi derived Indic scripts 
    Note that DANDA and DOUBLE_DANDA have the same Unicode codepoint for all Indic scripts 
    """
    if not is_supported_language(lang): 
        raise IndicNlpException('Language {}  not supported'.format(lang))
    o=get_offset(c,lang)
    return (o>=SCRIPT_OFFSET_START and o<SCRIPT_OFFSET_RANGE) \
            or ord(c)==li.DANDA or ord(c)==li.DOUBLE_DANDA

def in_coordinated_range_offset(c_offset): 
    """ 
    Applicable to Brahmi derived Indic scripts 
    """
    return  (c_offset>=li.COORDINATED_RANGE_START_INCLUSIVE and c_offset<=li.COORDINATED_RANGE_END_INCLUSIVE) 

def in_coordinated_range(c,lang):
    if not is_supported_language(lang): 
        raise IndicNlpException('Language {}  not supported'.format(lang))
    return in_coordinated_range_offset(get_offset(c,lang))

def get_phonetic_info(lang): 
    if not is_supported_language(lang): 
        raise IndicNlpException('Language {}  not supported'.format(lang))
    phonetic_data= ALL_PHONETIC_DATA if lang!=li.LC_TA else TAMIL_PHONETIC_DATA
    phonetic_vectors= ALL_PHONETIC_VECTORS if lang!=li.LC_TA else TAMIL_PHONETIC_VECTORS
   
    return (phonetic_data, phonetic_vectors)

def invalid_vector():
    ##  TODO: check if np datatype is correct?
    return np.array([0]*PHONETIC_VECTOR_LENGTH)

def get_phonetic_feature_vector(c,lang):

    offset=get_offset(c,lang) 

    if not in_coordinated_range_offset(offset): 
        return invalid_vector()

    phonetic_data, phonetic_vectors= get_phonetic_info(lang)

    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
        return invalid_vector()

    return phonetic_vectors[offset]

def get_phonetic_feature_vector_offset(offset,lang):

    if not in_coordinated_range_offset(offset): 
        return invalid_vector()

    phonetic_data, phonetic_vectors= get_phonetic_info(lang)

    if phonetic_data.iloc[offset]['Valid Vector Representation']==0: 
        return invalid_vector()

    return phonetic_vectors[offset]

### Unary operations on vectors 
def is_valid(v): 
    return np.sum(v)>0

def is_vowel(v): 
    return v[PVIDX_BT_VOWEL]==1 

def is_consonant(v): 
    return  v[PVIDX_BT_CONSONANT]==1 

def is_halant(v): 
    return  v[PVIDX_BT_HALANT]==1 

def is_nukta(v): 
    return  v[PVIDX_BT_NUKTA]==1 

def is_anusvaar(v): 
    return  v[PVIDX_BT_ANUSVAAR]==1 

def is_misc(v): 
    return v[PVIDX_BT_MISC]==1 

def is_dependent_vowel(v): 
    return is_vowel(v) and v[PVIDX_VSTAT_DEP]==1

def is_plosive(v): 
    return is_consonant(v) and get_property_vector(v,'consonant_type')[0]==1

### Binary operations on phonetic vectors

def or_vectors(v1,v2): 
    return np.array([ 1 if (b1+b2)>=1 else 0 for b1,b2 in zip(v1,v2) ])

def xor_vectors(v1,v2): 
    return np.array([ 1 if b1!=b2 else 0 for b1,b2 in zip(v1,v2) ])

### Getting properties from phonetic vectors 

def get_property_vector(v,prop_name): 
    return v[PV_PROP_RANGES[prop_name][0]:PV_PROP_RANGES[prop_name][1]]

def get_property_value(v,prop_name): 
    factor_bits=get_property_vector(v,prop_name).tolist()
    
    v=0
    c=1
    for b in factor_bits[::-1]: 
        v+=(c*b)
        c=c*2.0

    return int(v)

def lcsr_indic(srcw,tgtw,slang,tlang):
    """
    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.
    This works for Indic scripts by mapping both languages to a common script

    srcw: source language string
    tgtw: source language string
    slang: source language 
    tlang: target language 
    """    
    score_mat=np.zeros((len(srcw)+1,len(tgtw)+1))

    for si,sc in enumerate(srcw,1): 
        for ti,tc in enumerate(tgtw,1): 
            so=get_offset(sc,slang)
            to=get_offset(tc,tlang)

            if in_coordinated_range_offset(so) and in_coordinated_range_offset(to) and so==to: 
                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
            elif not (in_coordinated_range_offset(so) or in_coordinated_range_offset(to)) and sc==tc: 
                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
            else: 
                score_mat[si,ti]= max(
                    score_mat[si,ti-1],
                    score_mat[si-1,ti])

    return (score_mat[-1,-1]/float(max(len(srcw),len(tgtw))),float(len(srcw)),float(len(tgtw)))

def lcsr_any(srcw,tgtw):
    """
    LCSR computation if both languages have the same script
    """
    score_mat=np.zeros((len(srcw)+1,len(tgtw)+1))

    for si,sc in enumerate(srcw,1): 
        for ti,tc in enumerate(tgtw,1): 

            if sc==tc: 
                score_mat[si,ti]=score_mat[si-1,ti-1]+1.0
            else: 
                score_mat[si,ti]= max(
                    score_mat[si,ti-1],
                    score_mat[si-1,ti])

    return (score_mat[-1,-1]/float(max(len(srcw),len(tgtw))),float(len(srcw)),float(len(tgtw)))

def lcsr(srcw,tgtw,slang,tlang):
    """
    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.

    srcw: source language string
    tgtw: source language string
    slang: source language 
    tlang: target language 
    """

    if slang==tlang or not is_supported_language(slang) or not is_supported_language(tlang):
        return lcsr_any(srcw,tgtw,slang,tlang)
    else:  
        return lcsr_indic(srcw,tgtw)