libs/indic_nlp_library/indicnlp/script/indic_scripts.py

#
#  Copyright (c) 2013-present, Anoop Kunchukuttan
#  All rights reserved.
#
#  This source code is licensed under the MIT license found in the
#  LICENSE file in the root directory of this source tree.
#

import pandas as pd
import numpy as np
import os

from indicnlp import common
from indicnlp.common import IndicNlpException
from indicnlp import langinfo as li

###
# Phonetic Information about script characters
###

""" Phonetic data about all languages except Tamil """
ALL_PHONETIC_DATA = None

""" Phonetic data for Tamil """
TAMIL_PHONETIC_DATA = None

""" Phonetic vector for all languages except Tamil """
ALL_PHONETIC_VECTORS = None

""" Phonetic vector for Tamil """
TAMIL_PHONETIC_VECTORS = None

""" Length of phonetic vector """
PHONETIC_VECTOR_LENGTH = 38

""" Start offset for the phonetic feature vector in the phonetic data vector """
PHONETIC_VECTOR_START_OFFSET = 6

## PHONETIC PROPERTIES in order in which they occur in the vector
## This list must be in sync with the keys in the PV_PROP_RANGES dictionary
PV_PROP = [
    "basic_type",
    "vowel_length",
    "vowel_strength",
    "vowel_status",
    "consonant_type",
    "articulation_place",
    "aspiration",
    "voicing",
    "nasalization",
    "vowel_horizontal",
    "vowel_vertical",
    "vowel_roundness",
]

###
# Bit vector ranges for various properties
###

PV_PROP_RANGES = {
    "basic_type": [0, 6],
    "vowel_length": [6, 8],
    "vowel_strength": [8, 11],
    "vowel_status": [11, 13],
    "consonant_type": [13, 18],
    "articulation_place": [18, 23],
    "aspiration": [23, 25],
    "voicing": [25, 27],
    "nasalization": [27, 29],
    "vowel_horizontal": [29, 32],
    "vowel_vertical": [32, 36],
    "vowel_roundness": [36, 38],
}


####
# Indexes into the Phonetic Vector
####
PVIDX_BT_VOWEL = 0
PVIDX_BT_CONSONANT = 1
PVIDX_BT_NUKTA = 2
PVIDX_BT_HALANT = 3
PVIDX_BT_ANUSVAAR = 4
PVIDX_BT_MISC = 5
PVIDX_BT_S = PVIDX_BT_VOWEL
PVIDX_BT_E = PVIDX_BT_MISC + 1

PVIDX_VSTAT_DEP = 12

#####
# Unicode information about characters
#####

SCRIPT_OFFSET_START = 0
SCRIPT_OFFSET_RANGE = 0x80


def init():
    """
    To be called by library loader, do not call it in your program
    """

    global ALL_PHONETIC_DATA, ALL_PHONETIC_VECTORS, TAMIL_PHONETIC_DATA, TAMIL_PHONETIC_VECTORS, PHONETIC_VECTOR_LENGTH, PHONETIC_VECTOR_START_OFFSET

    ALL_PHONETIC_DATA = pd.read_csv(
        os.path.join(
            common.get_resources_path(), "script", "all_script_phonetic_data.csv"
        ),
        encoding="utf-8",
    )
    TAMIL_PHONETIC_DATA = pd.read_csv(
        os.path.join(
            common.get_resources_path(), "script", "tamil_script_phonetic_data.csv"
        ),
        encoding="utf-8",
    )

    ALL_PHONETIC_VECTORS = ALL_PHONETIC_DATA.iloc[
        :, PHONETIC_VECTOR_START_OFFSET:
    ].values
    TAMIL_PHONETIC_VECTORS = TAMIL_PHONETIC_DATA.iloc[
        :, PHONETIC_VECTOR_START_OFFSET:
    ].values

    PHONETIC_VECTOR_LENGTH = ALL_PHONETIC_VECTORS.shape[1]


def is_supported_language(lang):
    return lang in list(li.SCRIPT_RANGES.keys())


def get_offset(c, lang):
    if not is_supported_language(lang):
        raise IndicNlpException("Language {}  not supported".format(lang))
    return ord(c) - li.SCRIPT_RANGES[lang][0]


def offset_to_char(off, lang):
    """
    Applicable to Brahmi derived Indic scripts
    """
    if not is_supported_language(lang):
        raise IndicNlpException("Language {}  not supported".format(lang))
    return chr(off + li.SCRIPT_RANGES[lang][0])


def is_indiclang_char(c, lang):
    """
    Applicable to Brahmi derived Indic scripts
    Note that DANDA and DOUBLE_DANDA have the same Unicode codepoint for all Indic scripts
    """
    if not is_supported_language(lang):
        raise IndicNlpException("Language {}  not supported".format(lang))
    o = get_offset(c, lang)
    return (
        (o >= SCRIPT_OFFSET_START and o < SCRIPT_OFFSET_RANGE)
        or ord(c) == li.DANDA
        or ord(c) == li.DOUBLE_DANDA
    )


def in_coordinated_range_offset(c_offset):
    """
    Applicable to Brahmi derived Indic scripts
    """
    return (
        c_offset >= li.COORDINATED_RANGE_START_INCLUSIVE
        and c_offset <= li.COORDINATED_RANGE_END_INCLUSIVE
    )


def in_coordinated_range(c, lang):
    if not is_supported_language(lang):
        raise IndicNlpException("Language {}  not supported".format(lang))
    return in_coordinated_range_offset(get_offset(c, lang))


def get_phonetic_info(lang):
    if not is_supported_language(lang):
        raise IndicNlpException("Language {}  not supported".format(lang))
    phonetic_data = ALL_PHONETIC_DATA if lang != li.LC_TA else TAMIL_PHONETIC_DATA
    phonetic_vectors = (
        ALL_PHONETIC_VECTORS if lang != li.LC_TA else TAMIL_PHONETIC_VECTORS
    )

    return (phonetic_data, phonetic_vectors)


def invalid_vector():
    ##  TODO: check if np datatype is correct?
    return np.array([0] * PHONETIC_VECTOR_LENGTH)


def get_phonetic_feature_vector(c, lang):
    offset = get_offset(c, lang)

    if not in_coordinated_range_offset(offset):
        return invalid_vector()

    phonetic_data, phonetic_vectors = get_phonetic_info(lang)

    if phonetic_data.iloc[offset]["Valid Vector Representation"] == 0:
        return invalid_vector()

    return phonetic_vectors[offset]


def get_phonetic_feature_vector_offset(offset, lang):
    if not in_coordinated_range_offset(offset):
        return invalid_vector()

    phonetic_data, phonetic_vectors = get_phonetic_info(lang)

    if phonetic_data.iloc[offset]["Valid Vector Representation"] == 0:
        return invalid_vector()

    return phonetic_vectors[offset]


### Unary operations on vectors
def is_valid(v):
    return np.sum(v) > 0


def is_vowel(v):
    return v[PVIDX_BT_VOWEL] == 1


def is_consonant(v):
    return v[PVIDX_BT_CONSONANT] == 1


def is_halant(v):
    return v[PVIDX_BT_HALANT] == 1


def is_nukta(v):
    return v[PVIDX_BT_NUKTA] == 1


def is_anusvaar(v):
    return v[PVIDX_BT_ANUSVAAR] == 1


def is_misc(v):
    return v[PVIDX_BT_MISC] == 1


def is_dependent_vowel(v):
    return is_vowel(v) and v[PVIDX_VSTAT_DEP] == 1


def is_plosive(v):
    return is_consonant(v) and get_property_vector(v, "consonant_type")[0] == 1


### Binary operations on phonetic vectors


def or_vectors(v1, v2):
    return np.array([1 if (b1 + b2) >= 1 else 0 for b1, b2 in zip(v1, v2)])


def xor_vectors(v1, v2):
    return np.array([1 if b1 != b2 else 0 for b1, b2 in zip(v1, v2)])


### Getting properties from phonetic vectors


def get_property_vector(v, prop_name):
    return v[PV_PROP_RANGES[prop_name][0] : PV_PROP_RANGES[prop_name][1]]


def get_property_value(v, prop_name):
    factor_bits = get_property_vector(v, prop_name).tolist()

    v = 0
    c = 1
    for b in factor_bits[::-1]:
        v += c * b
        c = c * 2.0

    return int(v)


def lcsr_indic(srcw, tgtw, slang, tlang):
    """
    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.
    This works for Indic scripts by mapping both languages to a common script

    srcw: source language string
    tgtw: source language string
    slang: source language
    tlang: target language
    """
    score_mat = np.zeros((len(srcw) + 1, len(tgtw) + 1))

    for si, sc in enumerate(srcw, 1):
        for ti, tc in enumerate(tgtw, 1):
            so = get_offset(sc, slang)
            to = get_offset(tc, tlang)

            if (
                in_coordinated_range_offset(so)
                and in_coordinated_range_offset(to)
                and so == to
            ):
                score_mat[si, ti] = score_mat[si - 1, ti - 1] + 1.0
            elif (
                not (in_coordinated_range_offset(so) or in_coordinated_range_offset(to))
                and sc == tc
            ):
                score_mat[si, ti] = score_mat[si - 1, ti - 1] + 1.0
            else:
                score_mat[si, ti] = max(score_mat[si, ti - 1], score_mat[si - 1, ti])

    return (
        score_mat[-1, -1] / float(max(len(srcw), len(tgtw))),
        float(len(srcw)),
        float(len(tgtw)),
    )


def lcsr_any(srcw, tgtw):
    """
    LCSR computation if both languages have the same script
    """
    score_mat = np.zeros((len(srcw) + 1, len(tgtw) + 1))

    for si, sc in enumerate(srcw, 1):
        for ti, tc in enumerate(tgtw, 1):
            if sc == tc:
                score_mat[si, ti] = score_mat[si - 1, ti - 1] + 1.0
            else:
                score_mat[si, ti] = max(score_mat[si, ti - 1], score_mat[si - 1, ti])

    return (
        score_mat[-1, -1] / float(max(len(srcw), len(tgtw))),
        float(len(srcw)),
        float(len(tgtw)),
    )


def lcsr(srcw, tgtw, slang, tlang):
    """
    compute the Longest Common Subsequence Ratio (LCSR) between two strings at the character level.

    srcw: source language string
    tgtw: source language string
    slang: source language
    tlang: target language
    """

    if (
        slang == tlang
        or not is_supported_language(slang)
        or not is_supported_language(tlang)
    ):
        return lcsr_any(srcw, tgtw, slang, tlang)
    else:
        return lcsr_indic(srcw, tgtw)