Upload 6 files

normalize.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import re
+import regex
+
+from itertools import chain
+
+
+class MosesPunctNormalizer:
+    """
+    This is a Python port of the Moses punctuation normalizer from
+    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl
+    """
+
+    EXTRA_WHITESPACE = [  # lines 21 - 30
+        (r"\r", r""),
+        (r"\(", r" ("),
+        (r"\)", r") "),
+        (r" +", r" "),
+        (r"\) ([.!:?;,])", r")\g<1>"),
+        (r"\( ", r"("),
+        (r" \)", r")"),
+        (r"(\d) %", r"\g<1>%"),
+        (r" :", r":"),
+        (r" ;", r";"),
+    ]
+
+    NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')]  # lines 33 - 34
+
+    NORMALIZE_UNICODE = [  # lines 37 - 50
+        ("„", r'"'),
+        ("“", r'"'),
+        ("”", r'"'),
+        ("–", r"-"),
+        ("—", r" - "),
+        (r" +", r" "),
+        ("´", r"'"),
+        ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("‘", r"'"),
+        ("‚", r"'"),
+        ("’", r"'"),
+        (r"''", r'"'),
+        ("´´", r'"'),
+        ("…", r"..."),
+    ]
+
+    FRENCH_QUOTES = [  # lines 52 - 57
+        ("\u00A0«\u00A0", r'"'),
+        ("«\u00A0", r'"'),
+        ("«", r'"'),
+        ("\u00A0»\u00A0", r'"'),
+        ("\u00A0»", r'"'),
+        ("»", r'"'),
+    ]
+
+    HANDLE_PSEUDO_SPACES = [  # lines 59 - 67
+        ("\u00A0%", r"%"),
+        ("nº\u00A0", "nº "),
+        ("\u00A0:", r":"),
+        ("\u00A0ºC", " ºC"),
+        ("\u00A0cm", r" cm"),
+        ("\u00A0\\?", "?"),
+        ("\u00A0\\!", "!"),
+        ("\u00A0;", r";"),
+        (",\u00A0", r", "),
+        (r" +", r" "),
+    ]
+
+    EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')]
+
+    DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [
+        (r',"', r'",'),
+        (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'),  # don't fix period at end of sentence
+    ]
+
+    DE_ES_CZ_CS_FR = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"),
+    ]
+
+    OTHER = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"),
+    ]
+
+    # Regex substitutions from replace-unicode-punctuation.perl
+    # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
+    REPLACE_UNICODE_PUNCTUATION = [
+        ("，", ","),
+        (r"。\s*", ". "),
+        ("、", ","),
+        ("”", '"'),
+        ("“", '"'),
+        ("∶", ":"),
+        ("：", ":"),
+        ("？", "?"),
+        ("《", '"'),
+        ("》", '"'),
+        ("）", ")"),
+        ("！", "!"),
+        ("（", "("),
+        ("；", ";"),
+        ("」", '"'),
+        ("「", '"'),
+        ("０", "0"),
+        ("１", "1"),
+        ("２", "2"),
+        ("３", "3"),
+        ("４", "4"),
+        ("５", "5"),
+        ("６", "6"),
+        ("７", "7"),
+        ("８", "8"),
+        ("９", "9"),
+        (r"．\s*", ". "),
+        ("～", "~"),
+        ("’", "'"),
+        ("…", "..."),
+        ("━", "-"),
+        ("〈", "<"),
+        ("〉", ">"),
+        ("【", "["),
+        ("】", "]"),
+        ("％", "%"),
+    ]
+
+    def __init__(
+        self,
+        lang="en",
+        penn=True,
+        norm_quote_commas=True,
+        norm_numbers=True,
+        pre_replace_unicode_punct=False,
+        post_remove_control_chars=False,
+    ):
+        """
+        :param language: The two-letter language code.
+        :type lang: str
+        :param penn: Normalize Penn Treebank style quotations.
+        :type penn: bool
+        :param norm_quote_commas: Normalize quotations and commas
+        :type norm_quote_commas: bool
+        :param norm_numbers: Normalize numbers
+        :type norm_numbers: bool
+        """
+        self.substitutions = [
+            self.EXTRA_WHITESPACE,
+            self.NORMALIZE_UNICODE,
+            self.FRENCH_QUOTES,
+            self.HANDLE_PSEUDO_SPACES,
+        ]
+
+        if penn:  # Adds the penn substitutions after extra_whitespace regexes.
+            self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN)
+
+        if norm_quote_commas:
+            if lang == "en":
+                self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA)
+            elif lang in ["de", "es", "fr"]:
+                self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA)
+
+        if norm_numbers:
+            if lang in ["de", "es", "cz", "cs", "fr"]:
+                self.substitutions.append(self.DE_ES_CZ_CS_FR)
+            else:
+                self.substitutions.append(self.OTHER)
+
+        self.substitutions = list(chain(*self.substitutions))
+
+        self.pre_replace_unicode_punct = pre_replace_unicode_punct
+        self.post_remove_control_chars = post_remove_control_chars
+
+    def normalize(self, text):
+        """
+        Returns a string with normalized punctuation.
+        """
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.pre_replace_unicode_punct:
+            text = self.replace_unicode_punct(text)
+
+        # Actual normalization.
+        for regexp, substitution in self.substitutions:
+            # print(regexp, substitution)
+            text = re.sub(regexp, substitution, str(text))
+            # print(text)
+
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.post_remove_control_chars:
+            text = self.remove_control_chars(text)
+
+        return text.strip()
+
+    def replace_unicode_punct(self, text):
+        for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION:
+            text = re.sub(regexp, substitution, str(text))
+        return text
+
+    def remove_control_chars(self, text):
+        return regex.sub(r"\p{C}", "", text)

tokenization_korscideberta.py

@@ -0,0 +1,356 @@
+# coding=utf-8
+# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team and Jangwon Park
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Tokenization classes for KoBERT model """
+
+
+import logging
+import os
+import unicodedata
+from shutil import copyfile
+
+from transformers import PreTrainedTokenizer
+
+#2023. 7. 28. 형태소 분리(Mecab), 유니코드 정규화 추가
+from konlpy.tag import Mecab
+from unicode import join_jamos
+from normalize import MosesPunctNormalizer
+nor = MosesPunctNormalizer()
+
+# 유니코드 한글 시작 : 44032, 끝 : 55199
+BASE_CODE, CHOSUNG, JUNGSUNG = 44032, 588, 28
+# 초성 리스트. 0 ~ 18
+CHOSUNG_LIST = ['ㄱ', 'ㄲ', 'ㄴ', 'ㄷ', 'ㄸ', 'ㄹ', 'ㅁ', 'ㅂ', 'ㅃ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅉ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ']
+# 중성 리스트. 0 ~ 20
+JUNGSUNG_LIST = ['ㅏ', 'ㅐ', 'ㅑ', 'ㅒ', 'ㅓ', 'ㅔ', 'ㅕ', 'ㅖ', 'ㅗ', 'ㅘ', 'ㅙ', 'ㅚ', 'ㅛ', 'ㅜ', 'ㅝ', 'ㅞ', 'ㅟ', 'ㅠ', 'ㅡ', 'ㅢ', 'ㅣ']
+# 종성 리스트. 0 ~ 27 + 1(1개 없음)
+JONGSUNG_LIST = [' ', 'ㄱ', 'ㄲ', 'ㄳ', 'ㄴ', 'ㄵ', 'ㄶ', 'ㄷ', 'ㄹ', 'ㄺ', 'ㄻ', 'ㄼ', 'ㄽ', 'ㄾ', 'ㄿ', 'ㅀ', 'ㅁ', 'ㅂ', 'ㅄ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ']
+def splitjamo(string):
+    sp_list = list(string)
+    result = []
+    for keyword in sp_list:
+        # 한글 여부 check 후 분리
+        if re.match('.*[ㄱ-ㅎㅏ-ㅣ가-힣]+.*', keyword) is not None:
+            # 초성
+            char_code = ord(keyword) - BASE_CODE
+            char1 = int(char_code / CHOSUNG)
+            try:
+                result.append(CHOSUNG_LIST[char1])
+            except:
+                return string
+                #print("Err: "+str(char1))
+            # 중성
+            char2 = int((char_code - (CHOSUNG * char1)) / JUNGSUNG)
+            result.append(JUNGSUNG_LIST[char2])
+            # 종성
+            char3 = int((char_code - (CHOSUNG * char1) - (JUNGSUNG * char2)))
+            result.append(JONGSUNG_LIST[char3])
+        else:
+            result.append(keyword)
+    return result
+def has_coda(word):
+    return (ord(word[-1]) -44032)%28==0
+def _replace_unicode(line):
+    if(line==None):
+        return ""
+    line = line.replace("—",'-').replace("―","-").replace("–","-").replace("＂",'"').replace("＇","'").replace("‹","<").replace("›",">").replace("‚","'").replace("‛","'").replace("„",'"').replace("‟",'"').replace("«",'<').replace("»",'>').replace("˝",'"').replace("（",'(').replace("）",')').replace("『",'"').replace("』",'"').replace("“",'"').replace("”",'"').replace("‘","'").replace("’","'").replace("《","<").replace("》",">").replace("〈","<").replace("〉",">").replace("「","'").replace("」","'").replace("【","[").replace("】","]").replace("〔","[").replace("〕","]").replace("［","[").replace("］","]").replace("｛","{").replace("｝","}")
+    line=nor.replace_unicode_punct(line)
+    return line
+def _mecab(line):
+    mecab = Mecab()
+    #참고: VV동사 VA형용사 VX보조 용언 VCP긍정 지정사 VCN부정 지정사 JKS주격 조사 JKC보격 조사, … XSN명사 파생 접미사 XSV동사 파생 접미사 XSA형용사 파생 접미사 EP선어말 어미 EF종결 어미 EC연결 어미 ETN명사형 전성 어미 ETM관형형 전성 어미
+    
+    pdoc = []
+    morphs = []
+    
+    poss = mecab.pos(line)
+    for pos in poss:
+        morphs.append(pos[0])
+    '''
+    pdoc.append(" ".join(morphs))
+    return pdoc
+    '''
+    return " ".join(morphs)
+
+logger = logging.getLogger(__name__)
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "spm.model",
+    "vocab_txt": "vocab.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "monologg/kobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert/tokenizer_78b3253a26.model",
+        "monologg/kobert-lm": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert-lm/tokenizer_78b3253a26.model",
+        "monologg/distilkobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/distilkobert/tokenizer_78b3253a26.model",
+    },
+    "vocab_txt": {
+        "monologg/kobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert/vocab.txt",
+        "monologg/kobert-lm": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/kobert-lm/vocab.txt",
+        "monologg/distilkobert": "https://s3.amazonaws.com/models.huggingface.co/bert/monologg/distilkobert/vocab.txt",
+    },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "monologg/kobert": 512,
+    "monologg/kobert-lm": 512,
+    "monologg/distilkobert": 512,
+}
+
+PRETRAINED_INIT_CONFIGURATION = {
+    "monologg/kobert": {"do_lower_case": False},
+    "monologg/kobert-lm": {"do_lower_case": False},
+    "monologg/distilkobert": {"do_lower_case": False},
+}
+
+SPIECE_UNDERLINE = "▁"
+
+
+class DebertaV2Tokenizer(PreTrainedTokenizer):
+    """
+    SentencePiece based tokenizer. Peculiarities:
+        - requires `SentencePiece <https://github.com/google/sentencepiece>`_
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(
+        self,
+        vocab_file,
+        vocab_txt,
+        do_lower_case=False,
+        remove_space=True,
+        keep_accents=False,
+        unk_token="<unk>",
+        sep_token="<s>",
+        pad_token="<pad>",
+        cls_token="<cls>",
+        mask_token="<mask>",
+        **kwargs,
+    ):
+        super().__init__(
+            unk_token="<unk>",
+            sep_token=sep_token,
+            pad_token=pad_token,
+            cls_token=cls_token,
+            mask_token=mask_token,
+            **kwargs,
+        )
+
+        # Build vocab
+        self.token2idx = dict()
+        self.idx2token = []
+        with open(vocab_txt, "r", encoding="utf-8") as f:
+            for idx, token in enumerate(f):
+                token = token.strip()
+                self.token2idx[token] = idx
+                self.idx2token.append(token)
+
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            logger.warning(
+                "You need to install SentencePiece to use KoBertTokenizer: https://github.com/google/sentencepiece"
+                "pip install sentencepiece"
+            )
+
+        self.do_lower_case = do_lower_case
+        self.remove_space = remove_space
+        self.keep_accents = keep_accents
+        self.vocab_file = vocab_file
+        self.vocab_txt = vocab_txt
+
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(vocab_file)
+
+    @property
+    def vocab_size(self):
+        return len(self.idx2token)
+
+    def get_vocab(self):
+        return dict(self.token2idx, **self.added_tokens_encoder)
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["sp_model"] = None
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+        try:
+            import sentencepiece as spm
+        except ImportError:
+            logger.warning(
+                "You need to install SentencePiece to use KoBertTokenizer: https://github.com/google/sentencepiece"
+                "pip install sentencepiece"
+            )
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(self.vocab_file)
+
+    def preprocess_text(self, inputs):
+        if self.remove_space:
+            outputs = " ".join(inputs.strip().split())
+        else:
+            outputs = inputs
+        outputs = outputs.replace("``", '"').replace("''", '"')
+
+        if not self.keep_accents:
+            outputs = unicodedata.normalize("NFKD", outputs)
+            outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
+        if self.do_lower_case:
+            outputs = outputs.lower()
+
+        return outputs
+
+    def _tokenize(self, text):
+        """Tokenize a string."""
+        text = self.preprocess_text(text)
+        #print('text: '+text)
+        #logger.info("text ({}) ".format(text))
+        text = _replace_unicode(text) #유니코드 정규화
+        text = _mecab(text) #형태소 분리
+        #print('text: '+str(text))
+        #logger.info("text ({}) ".format(text))
+        pieces = self.sp_model.encode(text, out_type=str)
+        new_pieces = []
+        for piece in pieces:
+            if len(piece) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
+                cur_pieces = self.sp_model.EncodeAsPieces(piece[:-1].replace(SPIECE_UNDERLINE, ""))
+                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
+                    if len(cur_pieces[0]) == 1:
+                        cur_pieces = cur_pieces[1:]
+                    else:
+                        cur_pieces[0] = cur_pieces[0][1:]
+                cur_pieces.append(piece[-1])
+                new_pieces.extend(cur_pieces)
+            else:
+                new_pieces.append(piece)
+        '''
+        return_pieces = []
+        for n in new_pieces:
+            if(isinstance(n,list)):
+                for nn in n:
+                    return_pieces.append(nn)
+            else:
+                return_pieces.append(n)
+        return return_pieces
+        '''
+        return new_pieces
+        
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str/unicode) in an id using the vocab. """
+        return self.token2idx.get(token, self.token2idx[self.unk_token])
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (string/unicode) using the vocab."""
+        return self.idx2token[index]
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (strings for sub-words) in a single string."""
+        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
+        return out_string
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        by concatenating and adding special tokens.
+        A KoBERT sequence has the following format:
+            single sequence: [CLS] X [SEP]
+            pair of sequences: [CLS] A [SEP] B [SEP]
+        """
+        if token_ids_1 is None:
+            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+        cls = [self.cls_token_id]
+        sep = [self.sep_token_id]
+        return cls + token_ids_0 + sep + token_ids_1 + sep
+
+    def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
+        Args:
+            token_ids_0: list of ids (must not contain special tokens)
+            token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids
+                for sequence pairs
+            already_has_special_tokens: (default False) Set to True if the token list is already formated with
+                special tokens for the model
+        Returns:
+            A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            if token_ids_1 is not None:
+                raise ValueError(
+                    "You should not supply a second sequence if the provided sequence of "
+                    "ids is already formated with special tokens for the model."
+                )
+            return list(
+                map(
+                    lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0,
+                    token_ids_0,
+                )
+            )
+
+        if token_ids_1 is not None:
+            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
+        A KoBERT sequence pair mask has the following format:
+        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence
+        if token_ids_1 is None, only returns the first portion of the mask (0's).
+        """
+        sep = [self.sep_token_id]
+        cls = [self.cls_token_id]
+        if token_ids_1 is None:
+            return len(cls + token_ids_0 + sep) * [0]
+        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]
+
+    def save_vocabulary(self, save_directory):
+        """Save the sentencepiece vocabulary (copy original file) and special tokens file
+        to a directory.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error("Vocabulary path ({}) should be a directory".format(save_directory))
+            return
+
+        # 1. Save sentencepiece model
+        out_vocab_model = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_file"])
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_model):
+            copyfile(self.vocab_file, out_vocab_model)
+
+        # 2. Save vocab.txt
+        index = 0
+        out_vocab_txt = os.path.join(save_directory, VOCAB_FILES_NAMES["vocab_txt"])
+        with open(out_vocab_txt, "w", encoding="utf-8") as writer:
+            for token, token_index in sorted(self.token2idx.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        "Saving vocabulary to {}: vocabulary indices are not consecutive."
+                        " Please check that the vocabulary is not corrupted!".format(out_vocab_txt)
+                    )
+                    index = token_index
+                writer.write(token + "\n")
+                index += 1
+
+        return out_vocab_model, out_vocab_txt

tokenizer_config.json

@@ -0,0 +1,4 @@
+{
+  "do_lower_case": false,
+  "vocab_type": "spm"
+}

unicode.py

@@ -0,0 +1,279 @@
+__all__ = ["split_syllable_char", "split_syllables",
+           "join_jamos", "join_jamos_char",
+           "CHAR_INITIALS", "CHAR_MEDIALS", "CHAR_FINALS"]
+
+import itertools
+
+INITIAL = 0x001
+MEDIAL = 0x010
+FINAL = 0x100
+CHAR_LISTS = {
+    INITIAL: list(map(chr, [
+        0x3131, 0x3132, 0x3134, 0x3137, 0x3138, 0x3139,
+        0x3141, 0x3142, 0x3143, 0x3145, 0x3146, 0x3147,
+        0x3148, 0x3149, 0x314a, 0x314b, 0x314c, 0x314d,
+        0x314e
+    ])),
+    MEDIAL: list(map(chr, [
+        0x314f, 0x3150, 0x3151, 0x3152, 0x3153, 0x3154,
+        0x3155, 0x3156, 0x3157, 0x3158, 0x3159, 0x315a,
+        0x315b, 0x315c, 0x315d, 0x315e, 0x315f, 0x3160,
+        0x3161, 0x3162, 0x3163
+    ])),
+    FINAL: list(map(chr, [
+        0x3131, 0x3132, 0x3133, 0x3134, 0x3135, 0x3136,
+        0x3137, 0x3139, 0x313a, 0x313b, 0x313c, 0x313d,
+        0x313e, 0x313f, 0x3140, 0x3141, 0x3142, 0x3144,
+        0x3145, 0x3146, 0x3147, 0x3148, 0x314a, 0x314b,
+        0x314c, 0x314d, 0x314e
+    ]))
+}
+CHAR_INITIALS = CHAR_LISTS[INITIAL]
+CHAR_MEDIALS = CHAR_LISTS[MEDIAL]
+CHAR_FINALS = CHAR_LISTS[FINAL]
+CHAR_SETS = {k: set(v) for k, v in CHAR_LISTS.items()}
+CHARSET = set(itertools.chain(*CHAR_SETS.values()))
+CHAR_INDICES = {k: {c: i for i, c in enumerate(v)}
+                for k, v in CHAR_LISTS.items()}
+
+
+def is_hangul_syllable(c):
+    return 0xac00 <= ord(c) <= 0xd7a3  # Hangul Syllables
+
+
+def is_hangul_jamo(c):
+    return 0x1100 <= ord(c) <= 0x11ff  # Hangul Jamo
+
+
+def is_hangul_compat_jamo(c):
+    return 0x3130 <= ord(c) <= 0x318f  # Hangul Compatibility Jamo
+
+
+def is_hangul_jamo_exta(c):
+    return 0xa960 <= ord(c) <= 0xa97f  # Hangul Jamo Extended-A
+
+
+def is_hangul_jamo_extb(c):
+    return 0xd7b0 <= ord(c) <= 0xd7ff  # Hangul Jamo Extended-B
+
+
+def is_hangul(c):
+    return (is_hangul_syllable(c) or
+            is_hangul_jamo(c) or
+            is_hangul_compat_jamo(c) or
+            is_hangul_jamo_exta(c) or
+            is_hangul_jamo_extb(c))
+
+
+def is_supported_hangul(c):
+    return is_hangul_syllable(c) or is_hangul_compat_jamo(c)
+
+
+def check_hangul(c, jamo_only=False):
+    if not ((jamo_only or is_hangul_compat_jamo(c)) or is_supported_hangul(c)):
+        raise ValueError(f"'{c}' is not a supported hangul character. "
+                         f"'Hangul Syllables' (0xac00 ~ 0xd7a3) and "
+                         f"'Hangul Compatibility Jamos' (0x3130 ~ 0x318f) are "
+                         f"supported at the moment.")
+
+
+def get_jamo_type(c):
+    check_hangul(c)
+    assert is_hangul_compat_jamo(c), f"not a jamo: {ord(c):x}"
+    return sum(t for t, s in CHAR_SETS.items() if c in s)
+
+
+def split_syllable_char(c):
+    """
+    Splits a given korean syllable into its components. Each component is
+    represented by Unicode in 'Hangul Compatibility Jamo' range.
+
+    Arguments:
+        c: A Korean character.
+
+    Returns:
+        A triple (initial, medial, final) of Hangul Compatibility Jamos.
+        If no jamo corresponds to a position, `None` is returned there.
+
+    Example:
+        >>> split_syllable_char("안")
+        ("ㅇ", "ㅏ", "ㄴ")
+        >>> split_syllable_char("고")
+        ("ㄱ", "ㅗ", None)
+        >>> split_syllable_char("ㅗ")
+        (None, "ㅗ", None)
+        >>> split_syllable_char("ㅇ")
+        ("ㅇ", None, None)
+    """
+    check_hangul(c)
+    if len(c) != 1:
+        raise ValueError("Input string must have exactly one character.")
+
+    init, med, final = None, None, None
+    if is_hangul_syllable(c):
+        offset = ord(c) - 0xac00
+        x = (offset - offset % 28) // 28
+        init, med, final = x // 21, x % 21, offset % 28
+        if not final:
+            final = None
+        else:
+            final -= 1
+    else:
+        pos = get_jamo_type(c)
+        if pos & INITIAL == INITIAL:
+            pos = INITIAL
+        elif pos & MEDIAL == MEDIAL:
+            pos = MEDIAL
+        elif pos & FINAL == FINAL:
+            pos = FINAL
+        idx = CHAR_INDICES[pos][c]
+        if pos == INITIAL:
+            init = idx
+        elif pos == MEDIAL:
+            med = idx
+        else:
+            final = idx
+    return tuple(CHAR_LISTS[pos][idx] if idx is not None else None
+                 for pos, idx in
+                 zip([INITIAL, MEDIAL, FINAL], [init, med, final]))
+
+
+def split_syllables(s, ignore_err=True, pad=None):
+    """
+    Performs syllable-split on a string.
+
+    Arguments:
+        s (str): A string (possibly mixed with non-Hangul characters).
+        ignore_err (bool): If set False, it ensures that all characters in
+            the string are Hangul-splittable and throws a ValueError otherwise.
+            (default: True)
+        pad (str): Pad empty jamo positions (initial, medial, or final) with
+            `pad` character. This is useful for cases where fixed-length
+            strings are needed. (default: None)
+
+    Returns:
+        Hangul-split string
+
+    Example:
+        >>> split_syllables("안녕하세요")
+        "ㅇㅏㄴㄴㅕㅇ��ㅏㅅㅔㅇㅛ"
+        >>> split_syllables("안녕하세요~~", ignore_err=False)
+        ValueError: encountered an unsupported character: ~ (0x7e)
+        >>> split_syllables("안녕하세요ㅛ", pad="x")
+        'ㅇㅏㄴㄴㅕㅇㅎㅏxㅅㅔxㅇㅛxxㅛx'
+    """
+
+    def try_split(c):
+        try:
+            return split_syllable_char(c)
+        except ValueError:
+            if ignore_err:
+                return (c,)
+            raise ValueError(f"encountered an unsupported character: "
+                             f"{c} (0x{ord(c):x})")
+
+    s = map(try_split, s)
+    if pad is not None:
+        tuples = map(lambda x: tuple(pad if y is None else y for y in x), s)
+    else:
+        tuples = map(lambda x: filter(None, x), s)
+    return "".join(itertools.chain(*tuples))
+
+
+def join_jamos_char(init, med, final=None):
+    """
+    Combines jamos into a single syllable.
+
+    Arguments:
+        init (str): Initial jao.
+        med (str): Medial jamo.
+        final (str): Final jamo. If not supplied, the final syllable is made
+            without the final. (default: None)
+
+    Returns:
+        A Korean syllable.
+    """
+    chars = (init, med, final)
+    for c in filter(None, chars):
+        check_hangul(c, jamo_only=True)
+
+    idx = tuple(CHAR_INDICES[pos][c] if c is not None else c
+                for pos, c in zip((INITIAL, MEDIAL, FINAL), chars))
+    init_idx, med_idx, final_idx = idx
+    # final index must be shifted once as
+    # final index with 0 points to syllables without final
+    final_idx = 0 if final_idx is None else final_idx + 1
+    return chr(0xac00 + 28 * 21 * init_idx + 28 * med_idx + final_idx)
+
+
+def join_jamos(s, ignore_err=True):
+    """
+    Combines a sequence of jamos to produce a sequence of syllables.
+
+    Arguments:
+        s (str): A string (possible mixed with non-jamo characters).
+        ignore_err (bool): If set False, it will ensure that all characters
+            will be consumed for the making of syllables. It will throw a
+            ValueError when it fails to do so. (default: True)
+
+    Returns:
+        A string
+
+    Example:
+        >>> join_jamos("ㅇㅏㄴㄴㅕㅇㅎㅏㅅㅔㅇㅛ")
+        "안녕하세요"
+        >>> join_jamos("ㅇㅏㄴㄴㄴㅕㅇㅎㅏㅅㅔㅇㅛ")
+        "안ㄴ녕하세요"
+        >>> join_jamos()
+    """
+    last_t = 0
+    queue = []
+    new_string = ""
+
+    def flush(n=0):
+        new_queue = []
+        while len(queue) > n:
+            new_queue.append(queue.pop())
+        if len(new_queue) == 1:
+            if not ignore_err:
+                raise ValueError(f"invalid jamo character: {new_queue[0]}")
+            result = new_queue[0]
+        elif len(new_queue) >= 2:
+            try:
+                result = join_jamos_char(*new_queue)
+            except (ValueError, KeyError):
+                # Invalid jamo combination
+                if not ignore_err:
+                    raise ValueError(f"invalid jamo characters: {new_queue}")
+                result = "".join(new_queue)
+        else:
+            result = None
+        return result
+
+    for c in s:
+        if c not in CHARSET:
+            if queue:
+                new_c = flush() + c
+            else:
+                new_c = c
+            last_t = 0
+        else:
+            t = get_jamo_type(c)
+            new_c = None
+            if t & FINAL == FINAL:
+                if not (last_t == MEDIAL):
+                    new_c = flush()
+            elif t == INITIAL:
+                new_c = flush()
+            elif t == MEDIAL:
+                if last_t & INITIAL == INITIAL:
+                    new_c = flush(1)
+                else:
+                    new_c = flush()
+            last_t = t
+            queue.insert(0, c)
+        if new_c:
+            new_string += new_c
+    if queue:
+        new_string += flush()
+    return new_string