First commit

README.md

@@ -1,13 +1,14 @@
 ---
 title: MMS
-emoji: 👀
+emoji: 🌍
 colorFrom: pink
-colorTo: yellow
+colorTo: indigo
 sdk: gradio
-sdk_version: 3.35.2
+sdk_version: 3.32.0
 app_file: app.py
 pinned: false
-license: mit
+license: cc-by-nc-4.0
+duplicated_from: facebook/MMS
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,132 @@
+import gradio as gr
+import librosa
+from asr import transcribe, ASR_EXAMPLES, ASR_LANGUAGES, ASR_NOTE
+from tts import synthesize, TTS_EXAMPLES, TTS_LANGUAGES
+from lid import identify, LID_EXAMPLES
+
+
+demo = gr.Blocks()
+
+mms_select_source_trans = gr.Radio(
+    ["Record from Mic", "Upload audio"],
+    label="Audio input",
+    value="Record from Mic",
+)
+mms_mic_source_trans = gr.Audio(source="microphone", type="filepath", label="Use mic")
+mms_upload_source_trans = gr.Audio(
+    source="upload", type="filepath", label="Upload file", visible=False
+)
+mms_transcribe = gr.Interface(
+    fn=transcribe,
+    inputs=[
+        mms_select_source_trans,
+        mms_mic_source_trans,
+        mms_upload_source_trans,
+        gr.Dropdown(
+            [f"{k} ({v})" for k, v in ASR_LANGUAGES.items()],
+            label="Language",
+            value="eng English",
+        ),
+        # gr.Checkbox(label="Use Language Model (if available)", default=True),
+    ],
+    outputs="text",
+    examples=ASR_EXAMPLES,
+    title="Speech-to-text",
+    description=(
+        "Transcribe audio from a microphone or input file in your desired language."
+    ),
+    article=ASR_NOTE,
+    allow_flagging="never",
+)
+
+mms_synthesize = gr.Interface(
+    fn=synthesize,
+    inputs=[
+        gr.Text(label="Input text"),
+        gr.Dropdown(
+            [f"{k} ({v})" for k, v in TTS_LANGUAGES.items()],
+            label="Language",
+            value="eng English",
+        ),
+        gr.Slider(minimum=0.1, maximum=4.0, value=1.0, step=0.1, label="Speed"),
+    ],
+    outputs=[
+        gr.Audio(label="Generated Audio", type="numpy"),
+        gr.Text(label="Filtered text after removing OOVs"),
+    ],
+    examples=TTS_EXAMPLES,
+    title="Text-to-speech",
+    description=("Generate audio in your desired language from input text."),
+    allow_flagging="never",
+)
+
+mms_select_source_iden = gr.Radio(
+    ["Record from Mic", "Upload audio"],
+    label="Audio input",
+    value="Record from Mic",
+)
+mms_mic_source_iden = gr.Audio(source="microphone", type="filepath", label="Use mic")
+mms_upload_source_iden = gr.Audio(
+    source="upload", type="filepath", label="Upload file", visible=False
+)
+mms_identify = gr.Interface(
+    fn=identify,
+    inputs=[
+        mms_select_source_iden,
+        mms_mic_source_iden,
+        mms_upload_source_iden,
+    ],
+    outputs=gr.Label(num_top_classes=10),
+    examples=LID_EXAMPLES,
+    title="Language Identification",
+    description=("Identity the language of input audio."),
+    allow_flagging="never",
+)
+
+tabbed_interface = gr.TabbedInterface(
+    [mms_transcribe, mms_synthesize, mms_identify],
+    ["Speech-to-text", "Text-to-speech", "Language Identification"],
+)
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        "<p align='center' style='font-size: 20px;'>MMS: Scaling Speech Technology to 1000+ languages demo. See our <a href='https://ai.facebook.com/blog/multilingual-model-speech-recognition/'>blog post</a> and <a href='https://arxiv.org/abs/2305.13516'>paper</a>.</p>"
+    )
+    gr.HTML(
+        """<center>Click on the appropriate tab to explore Speech-to-text (ASR), Text-to-speech (TTS) and Language identification (LID) demos.   </center>"""
+    )
+    gr.HTML(
+        """<center><a href="https://huggingface.co/spaces/facebook/MMS?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a></center>"""
+    )
+
+    tabbed_interface.render()
+    mms_select_source_trans.change(
+        lambda x: [
+            gr.update(visible=True if x == "Record from Mic" else False),
+            gr.update(visible=True if x == "Upload audio" else False),
+        ],
+        inputs=[mms_select_source_trans],
+        outputs=[mms_mic_source_trans, mms_upload_source_trans],
+        queue=False,
+    )
+    mms_select_source_iden.change(
+        lambda x: [
+            gr.update(visible=True if x == "Record from Mic" else False),
+            gr.update(visible=True if x == "Upload audio" else False),
+        ],
+        inputs=[mms_select_source_iden],
+        outputs=[mms_mic_source_iden, mms_upload_source_iden],
+        queue=False,
+    )
+    gr.HTML(
+        """
+            <div class="footer" style="text-align:center">
+                <p>
+                    Model by <a href="https://ai.facebook.com" style="text-decoration: underline;" target="_blank">Meta AI</a> - Gradio Demo by 🤗 Hugging Face
+                </p>
+            </div>
+           """
+        )
+
+demo.queue(concurrency_count=10)
+demo.launch()

asr.py

@@ -0,0 +1,125 @@
+import librosa
+from transformers import Wav2Vec2ForCTC, AutoProcessor
+import torch
+import json
+
+from huggingface_hub import hf_hub_download
+from torchaudio.models.decoder import ctc_decoder
+
+ASR_SAMPLING_RATE = 16_000
+
+ASR_LANGUAGES = {}
+with open(f"data/asr/all_langs.tsv") as f:
+    for line in f:
+        iso, name = line.split(" ", 1)
+        ASR_LANGUAGES[iso] = name
+
+MODEL_ID = "facebook/mms-1b-all"
+
+processor = AutoProcessor.from_pretrained(MODEL_ID)
+model = Wav2Vec2ForCTC.from_pretrained(MODEL_ID)
+
+
+lm_decoding_config = {}
+# lm_decoding_configfile = hf_hub_download(
+#     repo_id="facebook/mms-cclms",
+#     filename="decoding_config.json",
+#     subfolder="mms-1b-all",
+# )
+
+# with open(lm_decoding_configfile) as f:
+#     lm_decoding_config = json.loads(f.read())
+
+# allow language model decoding for specific languages
+lm_decode_isos = ["eng"]
+
+
+def transcribe(
+    audio_source=None, microphone=None, file_upload=None, lang="eng (English)"
+):
+    if type(microphone) is dict:
+        # HACK: microphone variable is a dict when running on examples
+        microphone = microphone["name"]
+    audio_fp = (
+        file_upload if "upload" in str(audio_source or "").lower() else microphone
+    )
+    audio_samples = librosa.load(audio_fp, sr=ASR_SAMPLING_RATE, mono=True)[0]
+
+    lang_code = lang.split()[0]
+    processor.tokenizer.set_target_lang(lang_code)
+    model.load_adapter(lang_code)
+
+    inputs = processor(
+        audio_samples, sampling_rate=ASR_SAMPLING_RATE, return_tensors="pt"
+    )
+
+    # set device
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    elif (
+        hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+        and torch.backends.mps.is_built()
+    ):
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+
+    model.to(device)
+    inputs = inputs.to(device)
+
+    with torch.no_grad():
+        outputs = model(**inputs).logits
+
+    if lang_code not in lm_decoding_config or lang_code not in lm_decode_isos:
+        ids = torch.argmax(outputs, dim=-1)[0]
+        transcription = processor.decode(ids)
+    else:
+        decoding_config = lm_decoding_config[lang_code]
+
+        lm_file = hf_hub_download(
+            repo_id="facebook/mms-cclms",
+            filename=decoding_config["lmfile"].rsplit("/", 1)[1],
+            subfolder=decoding_config["lmfile"].rsplit("/", 1)[0],
+        )
+        token_file = hf_hub_download(
+            repo_id="facebook/mms-cclms",
+            filename=decoding_config["tokensfile"].rsplit("/", 1)[1],
+            subfolder=decoding_config["tokensfile"].rsplit("/", 1)[0],
+        )
+        lexicon_file = None
+        if decoding_config["lexiconfile"] is not None:
+            lexicon_file = hf_hub_download(
+                repo_id="facebook/mms-cclms",
+                filename=decoding_config["lexiconfile"].rsplit("/", 1)[1],
+                subfolder=decoding_config["lexiconfile"].rsplit("/", 1)[0],
+            )
+
+        beam_search_decoder = ctc_decoder(
+            lexicon=lexicon_file,
+            tokens=token_file,
+            lm=lm_file,
+            nbest=1,
+            beam_size=500,
+            beam_size_token=50,
+            lm_weight=float(decoding_config["lmweight"]),
+            word_score=float(decoding_config["wordscore"]),
+            sil_score=float(decoding_config["silweight"]),
+            blank_token="<s>",
+        )
+        beam_search_result = beam_search_decoder(outputs.to("cpu"))
+        transcription = " ".join(beam_search_result[0][0].words).strip()
+
+    return transcription
+
+
+ASR_EXAMPLES = [
+    [None, "assets/english.mp3", None, "eng (English)"],
+    # [None, "assets/tamil.mp3", None, "tam (Tamil)"],
+    # [None, "assets/burmese.mp3", None, "mya (Burmese)"],
+]
+
+ASR_NOTE = """
+The above demo uses beam-search decoding with LM for English and greedy decoding results for all other languages. 
+Checkout the instructions [here](https://huggingface.co/facebook/mms-1b-all) on how to run LM decoding for other languages.
+"""

data/asr/all_langs.tsv

@@ -0,0 +1,1198 @@
+abi Abidji
+abk Abkhaz
+abp Ayta, Abellen
+aca Achagua
+acd Gikyode
+ace Aceh
+acf Lesser Antillean French Creole
+ach Acholi
+acn Achang
+acr Achi
+acu Achuar-Shiwiar
+ade Adele
+adh Jopadhola
+adj Adioukrou
+adx Tibetan, Amdo
+aeu Akeu
+afr Afrikaans
+agd Agarabi
+agg Angor
+agn Agutaynen
+agr Awajún
+agu Awakateko
+agx Aghul
+aha Ahanta
+ahk Akha
+aia Arosi
+aka Akan
+akb Batak Angkola
+ake Akawaio
+akp Siwu
+alj Alangan
+alp Alune
+alt Altai, Southern
+alz Alur
+ame Yanesha’
+amf Hamer-Banna
+amh Amharic
+ami Amis
+amk Ambai
+ann Obolo
+any Anyin
+aoz Uab Meto
+apb Sa’a
+apr Arop-Lokep
+ara Arabic
+arl Arabela
+asa Asu
+asg Cishingini
+asm Assamese
+ast Asturian
+ata Pele-Ata
+atb Zaiwa
+atg Ivbie North-Okpela-Arhe
+ati Attié
+atq Aralle-Tabulahan
+ava Avar
+avn Avatime
+avu Avokaya
+awa Awadhi
+awb Awa
+ayo Ayoreo
+ayr Aymara, Central
+ayz Mai Brat
+azb Azerbaijani, South
+azg Amuzgo, San Pedro Amuzgos
+azj-script_cyrillic Azerbaijani, North
+azj-script_latin Azerbaijani, North
+azz Nahuatl, Highland Puebla
+bak Bashkort
+bam Bamanankan
+ban Bali
+bao Waimaha
+bas Basaa
+bav Vengo
+bba Baatonum
+bbb Barai
+bbc Batak Toba
+bbo Konabéré
+bcc-script_arabic Balochi, Southern
+bcc-script_latin Balochi, Southern
+bcl Bikol, Central
+bcw Bana
+bdg Bonggi
+bdh Baka
+bdq Bahnar
+bdu Oroko
+bdv Bodo Parja
+beh Biali
+bel Belarusian
+bem Bemba
+ben Bengali
+bep Behoa
+bex Jur Modo
+bfa Bari
+bfo Birifor, Malba
+bfy Bagheli
+bfz Pahari, Mahasu
+bgc Haryanvi
+bgq Bagri
+bgr Chin, Bawm
+bgt Bughotu
+bgw Bhatri
+bha Bharia
+bht Bhattiyali
+bhz Bada
+bib Bisa
+bim Bimoba
+bis Bislama
+biv Birifor, Southern
+bjr Binumarien
+bjv Bedjond
+bjw Bakwé
+bjz Baruga
+bkd Binukid
+bkv Bekwarra
+blh Kuwaa
+blt Tai Dam
+blx Ayta, Mag-Indi
+blz Balantak
+bmq Bomu
+bmr Muinane
+bmu Somba-Siawari
+bmv Bum
+bng Benga
+bno Bantoanon
+bnp Bola
+boa Bora
+bod Tibetan, Central
+boj Anjam
+bom Berom
+bor Borôro
+bos Bosnian
+bov Tuwuli
+box Buamu
+bpr Blaan, Koronadal
+bps Blaan, Sarangani
+bqc Boko
+bqi Bakhtiâri
+bqj Bandial
+bqp Bisã
+bre Breton
+bru Bru, Eastern
+bsc Oniyan
+bsq Bassa
+bss Akoose
+btd Batak Dairi
+bts Batak Simalungun
+btt Bete-Bendi
+btx Batak Karo
+bud Ntcham
+bul Bulgarian
+bus Bokobaru
+bvc Baelelea
+bvz Bauzi
+bwq Bobo Madaré, Southern
+bwu Buli
+byr Yipma
+bzh Buang, Mapos
+bzi Bisu
+bzj Belize English Creole
+caa Ch’orti’
+cab Garifuna
+cac-dialect_sanmateoixtatan Chuj
+cac-dialect_sansebastiancoatan Chuj
+cak-dialect_central Kaqchikel
+cak-dialect_santamariadejesus Kaqchikel
+cak-dialect_santodomingoxenacoj Kaqchikel
+cak-dialect_southcentral Kaqchikel
+cak-dialect_western Kaqchikel
+cak-dialect_yepocapa Kaqchikel
+cap Chipaya
+car Carib
+cas Tsimané
+cat Catalan
+cax Chiquitano
+cbc Carapana
+cbi Chachi
+cbr Kakataibo-Kashibo
+cbs Kashinawa
+cbt Shawi
+cbu Kandozi-Chapra
+cbv Cacua
+cce Chopi
+cco Chinantec, Comaltepec
+cdj Churahi
+ceb Cebuano
+ceg Chamacoco
+cek Chin, Eastern Khumi
+ces Czech
+cfm Chin, Falam
+cgc Kagayanen
+che Chechen
+chf Chontal, Tabasco
+chv Chuvash
+chz Chinantec, Ozumacín
+cjo Ashéninka, Pajonal
+cjp Cabécar
+cjs Shor
+ckb Kurdish, Central
+cko Anufo
+ckt Chukchi
+cla Ron
+cle Chinantec, Lealao
+cly Chatino, Eastern Highland
+cme Cerma
+cmn-script_simplified Chinese, Mandarin
+cmo-script_khmer Mnong, Central
+cmo-script_latin Mnong, Central
+cmr Mro-Khimi
+cnh Chin, Hakha
+cni Asháninka
+cnl Chinantec, Lalana
+cnt Chinantec, Tepetotutla
+coe Koreguaje
+cof Tsafiki
+cok Cora, Santa Teresa
+con Cofán
+cot Caquinte
+cou Wamey
+cpa Chinantec, Palantla
+cpb Ashéninka, Ucayali-Yurúa
+cpu Ashéninka, Pichis
+crh Crimean Tatar
+crk-script_latin Cree, Plains
+crk-script_syllabics Cree, Plains
+crn Cora, El Nayar
+crq Chorote, Iyo’wujwa
+crs Seychelles French Creole
+crt Chorote, Iyojwa’ja
+csk Jola-Kasa
+cso Chinantec, Sochiapam
+ctd Chin, Tedim
+ctg Chittagonian
+cto Embera Catío
+ctu Chol
+cuc Chinantec, Usila
+cui Cuiba
+cuk Kuna, San Blas
+cul Kulina
+cwa Kabwa
+cwe Kwere
+cwt Kuwaataay
+cya Chatino, Nopala
+cym Welsh
+daa Dangaléat
+dah Gwahatike
+dan Danish
+dar Dargwa
+dbj Ida’an
+dbq Daba
+ddn Dendi
+ded Dedua
+des Desano
+deu German, Standard
+dga Dagaare, Southern
+dgi Dagara, Northern
+dgk Dagba
+dgo Dogri
+dgr Tlicho
+dhi Dhimal
+did Didinga
+dig Chidigo
+dik Dinka, Southwestern
+dip Dinka, Northeastern
+div Maldivian
+djk Aukan
+dnj-dialect_blowowest Dan
+dnj-dialect_gweetaawueast Dan
+dnt Dani, Mid Grand Valley
+dnw Dani, Western
+dop Lukpa
+dos Dogosé
+dsh Daasanach
+dso Desiya
+dtp Kadazan Dusun
+dts Dogon, Toro So
+dug Chiduruma
+dwr Dawro
+dyi Sénoufo, Djimini
+dyo Jola-Fonyi
+dyu Jula
+dzo Dzongkha
+eip Lik
+eka Ekajuk
+ell Greek
+emp Emberá, Northern
+enb Markweeta
+eng English
+enx Enxet
+epo Esperanto
+ese Ese Ejja
+ess Yupik, Saint Lawrence Island
+est Estonian
+eus Basque
+evn Evenki
+ewe Éwé
+eza Ezaa
+fal Fali, South
+fao Faroese
+far Fataleka
+fas Persian
+fij Fijian
+fin Finnish
+flr Fuliiru
+fmu Muria, Far Western
+fon Fon
+fra French
+frd Fordata
+fry Frisian
+ful Fulah
+gag-script_cyrillic Gagauz
+gag-script_latin Gagauz
+gai Mbore
+gam Kandawo
+gau Gadaba, Mudhili
+gbi Galela
+gbk Gaddi
+gbm Garhwali
+gbo Grebo, Northern
+gde Gude
+geb Kire
+gej Gen
+gil Kiribati
+gjn Gonja
+gkn Gokana
+gld Nanai
+gle Irish
+glg Galician
+glk Gilaki
+gmv Gamo
+gna Kaansa
+gnd Zulgo-Gemzek
+gng Ngangam
+gof-script_latin Gofa
+gog Gogo
+gor Gorontalo
+gqr Gor
+grc Greek, Ancient
+gri Ghari
+grn Guarani
+grt Garo
+gso Gbaya, Southwest
+gub Guajajára
+guc Wayuu
+gud Dida, Yocoboué
+guh Guahibo
+guj Gujarati
+guk Gumuz
+gum Misak
+guo Guayabero
+guq Aché
+guu Yanomamö
+gux Gourmanchéma
+gvc Wanano
+gvl Gulay
+gwi Gwich’in
+gwr Gwere
+gym Ngäbere
+gyr Guarayu
+had Hatam
+hag Hanga
+hak Chinese, Hakka
+hap Hupla
+hat Haitian Creole
+hau Hausa
+hay Haya
+heb Hebrew
+heh Hehe
+hif Hindi, Fiji
+hig Kamwe
+hil Hiligaynon
+hin Hindi
+hlb Halbi
+hlt Chin, Matu
+hne Chhattisgarhi
+hnn Hanunoo
+hns Hindustani, Sarnami
+hoc Ho
+hoy Holiya
+hrv Croatian
+hsb Sorbian, Upper
+hto Witoto, Minika
+hub Wampís
+hui Huli
+hun Hungarian
+hus-dialect_centralveracruz Huastec
+hus-dialect_westernpotosino Huastec
+huu Witoto, Murui
+huv Huave, San Mateo del Mar
+hvn Hawu
+hwc Hawaii Pidgin
+hye Armenian
+hyw Armenian, Western
+iba Iban
+ibo Igbo
+icr Islander English Creole
+idd Ede Idaca
+ifa Ifugao, Amganad
+ifb Ifugao, Batad
+ife Ifè
+ifk Ifugao, Tuwali
+ifu Ifugao, Mayoyao
+ify Kallahan, Keley-i
+ign Ignaciano
+ikk Ika
+ilb Ila
+ilo Ilocano
+imo Imbongu
+ina Interlingua (International Auxiliary Language Association)
+inb Inga
+ind Indonesian
+iou Tuma-Irumu
+ipi Ipili
+iqw Ikwo
+iri Rigwe
+irk Iraqw
+isl Icelandic
+ita Italian
+itl Itelmen
+itv Itawit
+ixl-dialect_sangasparchajul Ixil
+ixl-dialect_sanjuancotzal Ixil
+ixl-dialect_santamarianebaj Ixil
+izr Izere
+izz Izii
+jac Jakalteko
+jam Jamaican English Creole
+jav Javanese
+jbu Jukun Takum
+jen Dza
+jic Tol
+jiv Shuar
+jmc Machame
+jmd Yamdena
+jpn Japanese
+jun Juang
+juy Juray
+jvn Javanese, Suriname
+kaa Karakalpak
+kab Amazigh
+kac Jingpho
+kak Kalanguya
+kam Kamba
+kan Kannada
+kao Xaasongaxango
+kaq Capanahua
+kat Georgian
+kay Kamayurá
+kaz Kazakh
+kbo Keliko
+kbp Kabiyè
+kbq Kamano
+kbr Kafa
+kby Kanuri, Manga
+kca Khanty
+kcg Tyap
+kdc Kutu
+kde Makonde
+kdh Tem
+kdi Kumam
+kdj Ng’akarimojong
+kdl Tsikimba
+kdn Kunda
+kdt Kuay
+kea Kabuverdianu
+kek Q’eqchi’
+ken Kenyang
+keo Kakwa
+ker Kera
+key Kupia
+kez Kukele
+kfb Kolami, Northwestern
+kff-script_telugu Koya
+kfw Naga, Kharam
+kfx Pahari, Kullu
+khg Tibetan, Khams
+khm Khmer
+khq Songhay, Koyra Chiini
+kia Kim
+kij Kilivila
+kik Gikuyu
+kin Kinyarwanda
+kir Kyrgyz
+kjb Q’anjob’al
+kje Kisar
+kjg Khmu
+kjh Khakas
+kki Kagulu
+kkj Kako
+kle Kulung
+klu Klao
+klv Maskelynes
+klw Tado
+kma Konni
+kmd Kalinga, Majukayang
+kml Kalinga, Tanudan
+kmr-script_arabic Kurdish, Northern
+kmr-script_cyrillic Kurdish, Northern
+kmr-script_latin Kurdish, Northern
+kmu Kanite
+knb Kalinga, Lubuagan
+kne Kankanaey
+knf Mankanya
+knj Akateko
+knk Kuranko
+kno Kono
+kog Kogi
+kor Korean
+kpq Korupun-Sela
+kps Tehit
+kpv Komi-Zyrian
+kpy Koryak
+kpz Kupsapiiny
+kqe Kalagan
+kqp Kimré
+kqr Kimaragang
+kqy Koorete
+krc Karachay-Balkar
+kri Krio
+krj Kinaray-a
+krl Karelian
+krr Krung
+krs Gbaya
+kru Kurux
+ksb Shambala
+ksr Borong
+kss Kisi, Southern
+ktb Kambaata
+ktj Krumen, Plapo
+kub Kutep
+kue Kuman
+kum Kumyk
+kus Kusaal
+kvn Kuna, Border
+kvw Wersing
+kwd Kwaio
+kwf Kwara’ae
+kwi Awa-Cuaiquer
+kxc Konso
+kxf Kawyaw
+kxm Khmer, Northern
+kxv Kuvi
+kyb Kalinga, Butbut
+kyc Kyaka
+kyf Kouya
+kyg Keyagana
+kyo Klon
+kyq Kenga
+kyu Kayah, Western
+kyz Kayabí
+kzf Kaili, Da’a
+lac Lacandon
+laj Lango
+lam Lamba
+lao Lao
+las Lama
+lat Latin
+lav Latvian
+law Lauje
+lbj Ladakhi
+lbw Tolaki
+lcp Lawa, Western
+lee Lyélé
+lef Lelemi
+lem Nomaande
+lew Kaili, Ledo
+lex Luang
+lgg Lugbara
+lgl Wala
+lhu Lahu
+lia Limba, West-Central
+lid Nyindrou
+lif Limbu
+lin Lingala
+lip Sekpele
+lis Lisu
+lit Lithuanian
+lje Rampi
+ljp Lampung Api
+llg Lole
+lln Lele
+lme Pévé
+lnd Lundayeh
+lns Lamnso’
+lob Lobi
+lok Loko
+lom Loma
+lon Lomwe, Malawi
+loq Lobala
+lsi Lacid
+lsm Saamya-Gwe
+ltz Luxembourgish
+luc Aringa
+lug Ganda
+luo Dholuo
+lwo Luwo
+lww Lewo
+lzz Laz
+maa-dialect_sanantonio Mazatec, San Jerónimo Tecóatl
+maa-dialect_sanjeronimo Mazatec, San Jerónimo Tecóatl
+mad Madura
+mag Magahi
+mah Marshallese
+mai Maithili
+maj Mazatec, Jalapa de Díaz
+mak Makasar
+mal Malayalam
+mam-dialect_central Mam
+mam-dialect_northern Mam
+mam-dialect_southern Mam
+mam-dialect_western Mam
+maq Mazatec, Chiquihuitlán
+mar Marathi
+maw Mampruli
+maz Mazahua, Central
+mbb Manobo, Western Bukidnon
+mbc Macushi
+mbh Mangseng
+mbj Nadëb
+mbt Manobo, Matigsalug
+mbu Mbula-Bwazza
+mbz Mixtec, Amoltepec
+mca Maka
+mcb Matsigenka
+mcd Sharanahua
+mco Mixe, Coatlán
+mcp Makaa
+mcq Ese
+mcu Mambila, Cameroon
+mda Mada
+mdf Moksha
+mdv Mixtec, Santa Lucía Monteverde
+mdy Male
+med Melpa
+mee Mengen
+mej Meyah
+men Mende
+meq Merey
+met Mato
+mev Maan
+mfe Morisyen
+mfh Matal
+mfi Wandala
+mfk Mofu, North
+mfq Moba
+mfy Mayo
+mfz Mabaan
+mgd Moru
+mge Mango
+mgh Makhuwa-Meetto
+mgo Meta’
+mhi Ma’di
+mhr Mari, Meadow
+mhu Digaro-Mishmi
+mhx Lhao Vo
+mhy Ma’anyan
+mib Mixtec, Atatlahuca
+mie Mixtec, Ocotepec
+mif Mofu-Gudur
+mih Mixtec, Chayuco
+mil Mixtec, Peñoles
+mim Mixtec, Alacatlatzala
+min Minangkabau
+mio Mixtec, Pinotepa Nacional
+mip Mixtec, Apasco-Apoala
+miq Mískito
+mit Mixtec, Southern Puebla
+miy Mixtec, Ayutla
+miz Mixtec, Coatzospan
+mjl Mandeali
+mjv Mannan
+mkd Macedonian
+mkl Mokole
+mkn Malay, Kupang
+mlg Malagasy
+mlt Maltese
+mmg Ambrym, North
+mnb Muna
+mnf Mundani
+mnk Mandinka
+mnw Mon
+mnx Sougb
+moa Mwan
+mog Mongondow
+mon Mongolian
+mop Maya, Mopán
+mor Moro
+mos Mòoré
+mox Molima
+moz Mukulu
+mpg Marba
+mpm Mixtec, Yosondúa
+mpp Migabac
+mpx Misima-Panaeati
+mqb Mbuko
+mqf Momuna
+mqj Mamasa
+mqn Moronene
+mri Maori
+mrw Maranao
+msy Aruamu
+mtd Mualang
+mtj Moskona
+mto Mixe, Totontepec
+muh Mündü
+mup Malvi
+mur Murle
+muv Muthuvan
+muy Muyang
+mvp Duri
+mwq Chin, Müün
+mwv Mentawai
+mxb Mixtec, Tezoatlán
+mxq Mixe, Juquila
+mxt Mixtec, Jamiltepec
+mxv Mixtec, Metlatónoc
+mya Burmese
+myb Mbay
+myk Sénoufo, Mamara
+myl Moma
+myv Erzya
+myx Masaaba
+myy Macuna
+mza Mixtec, Santa María Zacatepec
+mzi Mazatec, Ixcatlán
+mzj Manya
+mzk Mambila, Nigeria
+mzm Mumuye
+mzw Deg
+nab Nambikuára, Southern
+nag Nagamese
+nan Chinese, Min Nan
+nas Naasioi
+naw Nawuri
+nca Iyo
+nch Nahuatl, Central Huasteca
+ncj Nahuatl, Northern Puebla
+ncl Nahuatl, Michoacán
+ncu Chumburung
+ndj Ndamba
+ndp Kebu
+ndv Ndut
+ndy Lutos
+ndz Ndogo
+neb Toura
+new Newar
+nfa Dhao
+nfr Nafaanra
+nga Ngbaka
+ngl Lomwe
+ngp Ngulu
+ngu Nahuatl, Guerrero
+nhe Nahuatl, Eastern Huasteca
+nhi Nahuatl, Zacatlán-Ahuacatlán-Tepetzintla
+nhu Noone
+nhw Nahuatl, Western Huasteca
+nhx Nahuatl, Isthmus-Mecayapan
+nhy Nahuatl, Northern Oaxaca
+nia Nias
+nij Ngaju
+nim Nilamba
+nin Ninzo
+nko Nkonya
+nlc Nalca
+nld Dutch
+nlg Gela
+nlk Yali, Ninia
+nmz Nawdm
+nnb Nande
+nno Norwegian Nynorsk
+nnq Ngindo
+nnw Nuni, Southern
+noa Woun Meu
+nob Norwegian Bokmål
+nod Thai, Northern
+nog Nogai
+not Nomatsigenga
+npi Nepali
+npl Nahuatl, Southeastern Puebla
+npy Napu
+nso Sotho, Northern
+nst Naga, Tangshang
+nsu Nahuatl, Sierra Negra
+ntm Nateni
+ntr Delo
+nuj Nyole
+nus Nuer
+nuz Nahuatl, Tlamacazapa
+nwb Nyabwa
+nxq Naxi
+nya Chichewa
+nyf Kigiryama
+nyn Nyankore
+nyo Nyoro
+nyy Nyakyusa-Ngonde
+nzi Nzema
+obo Manobo, Obo
+oci Occitan
+ojb-script_latin Ojibwa, Northwestern
+ojb-script_syllabics Ojibwa, Northwestern
+oku Oku
+old Mochi
+omw Tairora, South
+onb Lingao
+ood Tohono O’odham
+orm Oromo
+ory Odia
+oss Ossetic
+ote Otomi, Mezquital
+otq Otomi, Querétaro
+ozm Koonzime
+pab Parecís
+pad Paumarí
+pag Pangasinan
+pam Kapampangan
+pan Punjabi, Eastern
+pao Paiute, Northern
+pap Papiamentu
+pau Palauan
+pbb Nasa
+pbc Patamona
+pbi Parkwa
+pce Palaung, Ruching
+pcm Pidgin, Nigerian
+peg Pengo
+pez Penan, Eastern
+pib Yine
+pil Yom
+pir Piratapuyo
+pis Pijin
+pjt Pitjantjatjara
+pkb Kipfokomo
+pls Popoloca, San Marcos Tlacoyalco
+plw Palawano, Brooke’s Point
+pmf Pamona
+pny Pinyin
+poh-dialect_eastern Poqomchi’
+poh-dialect_western Poqomchi’
+poi Popoluca, Highland
+pol Polish
+por Portuguese
+poy Pogolo
+ppk Uma
+pps Popoloca, San Luís Temalacayuca
+prf Paranan
+prk Wa, Parauk
+prt Prai
+pse Malay, Central
+pss Kaulong
+ptu Bambam
+pui Puinave
+pus Pushto
+pwg Gapapaiwa
+pww Karen, Pwo Northern
+pxm Mixe, Quetzaltepec
+qub Quechua, Huallaga
+quc-dialect_central K’iche’
+quc-dialect_east K’iche’
+quc-dialect_north K’iche’
+quf Quechua, Lambayeque
+quh Quechua, South Bolivian
+qul Quechua, North Bolivian
+quw Quichua, Tena Lowland
+quy Quechua, Ayacucho
+quz Quechua, Cusco
+qvc Quechua, Cajamarca
+qve Quechua, Eastern Apurímac
+qvh Quechua, Huamalíes-Dos de Mayo Huánuco
+qvm Quechua, Margos-Yarowilca-Lauricocha
+qvn Quechua, North Junín
+qvo Quichua, Napo
+qvs Quechua, San Martín
+qvw Quechua, Huaylla Wanca
+qvz Quichua, Northern Pastaza
+qwh Quechua, Huaylas Ancash
+qxh Quechua, Panao
+qxl Quichua, Salasaca Highland
+qxn Quechua, Northern Conchucos Ancash
+qxo Quechua, Southern Conchucos
+qxr Quichua, Cañar Highland
+rah Rabha
+rai Ramoaaina
+rap Rapa Nui
+rav Sampang
+raw Rawang
+rej Rejang
+rel Rendille
+rgu Rikou
+rhg Rohingya
+rif-script_arabic Tarifit
+rif-script_latin Tarifit
+ril Riang Lang
+rim Nyaturu
+rjs Rajbanshi
+rkt Rangpuri
+rmc-script_cyrillic Romani, Carpathian
+rmc-script_latin Romani, Carpathian
+rmo Romani, Sinte
+rmy-script_cyrillic Romani, Vlax
+rmy-script_latin Romani, Vlax
+rng Ronga
+rnl Ranglong
+roh-dialect_sursilv Romansh
+roh-dialect_vallader Romansh
+rol Romblomanon
+ron Romanian
+rop Kriol
+rro Waima
+rub Gungu
+ruf Luguru
+rug Roviana
+run Rundi
+rus Russian
+sab Buglere
+sag Sango
+sah Yakut
+saj Sahu
+saq Samburu
+sas Sasak
+sat Santhali
+sba Ngambay
+sbd Samo, Southern
+sbl Sambal, Botolan
+sbp Sangu
+sch Sakachep
+sck Sadri
+sda Toraja-Sa’dan
+sea Semai
+seh Sena
+ses Songhay, Koyraboro Senni
+sey Paicoca
+sgb Ayta, Mag-antsi
+sgj Surgujia
+sgw Sebat Bet Gurage
+shi Tachelhit
+shk Shilluk
+shn Shan
+sho Shanga
+shp Shipibo-Conibo
+sid Sidamo
+sig Paasaal
+sil Sisaala, Tumulung
+sja Epena
+sjm Mapun
+sld Sissala
+slk Slovak
+slu Selaru
+slv Slovene
+sml Sama, Central
+smo Samoan
+sna Shona
+snd Sindhi
+sne Bidayuh, Bau
+snn Siona
+snp Siane
+snw Selee
+som Somali
+soy Miyobe
+spa Spanish
+spp Sénoufo, Supyire
+spy Sabaot
+sqi Albanian
+sri Siriano
+srm Saramaccan
+srn Sranan Tongo
+srp-script_cyrillic Serbian
+srp-script_latin Serbian
+srx Sirmauri
+stn Owa
+stp Tepehuan, Southeastern
+suc Subanon, Western
+suk Sukuma
+sun Sunda
+sur Mwaghavul
+sus Susu
+suv Puroik
+suz Sunwar
+swe Swedish
+swh Swahili
+sxb Suba
+sxn Sangir
+sya Siang
+syl Sylheti
+sza Semelai
+tac Tarahumara, Western
+taj Tamang, Eastern
+tam Tamil
+tao Yami
+tap Taabwa
+taq Tamasheq
+tat Tatar
+tav Tatuyo
+tbc Takia
+tbg Tairora, North
+tbk Tagbanwa, Calamian
+tbl Tboli
+tby Tabaru
+tbz Ditammari
+tca Ticuna
+tcc Datooga
+tcs Torres Strait Creole
+tcz Chin, Thado
+tdj Tajio
+ted Krumen, Tepo
+tee Tepehua, Huehuetla
+tel Telugu
+tem Themne
+teo Ateso
+ter Terêna
+tes Tengger
+tew Tewa
+tex Tennet
+tfr Teribe
+tgj Tagin
+tgk Tajik
+tgl Tagalog
+tgo Sudest
+tgp Tangoa
+tha Thai
+thk Kitharaka
+thl Tharu, Dangaura
+tih Murut, Timugon
+tik Tikar
+tir Tigrigna
+tkr Tsakhur
+tlb Tobelo
+tlj Talinga-Bwisi
+tly Talysh
+tmc Tumak
+tmf Toba-Maskoy
+tna Tacana
+tng Tobanga
+tnk Kwamera
+tnn Tanna, North
+tnp Whitesands
+tnr Ménik
+tnt Tontemboan
+tob Toba
+toc Totonac, Coyutla
+toh Tonga
+tom Tombulu
+tos Totonac, Highland
+tpi Tok Pisin
+tpm Tampulma
+tpp Tepehua, Pisaflores
+tpt Tepehua, Tlachichilco
+trc Triqui, Copala
+tri Trió
+trn Trinitario
+trs Triqui, Chicahuaxtla
+tso Tsonga
+tsz Purepecha
+ttc Tektiteko
+tte Bwanabwana
+ttq-script_tifinagh Tamajaq, Tawallammat
+tue Tuyuca
+tuf Tunebo, Central
+tuk-script_arabic Turkmen
+tuk-script_latin Turkmen
+tuo Tucano
+tur Turkish
+tvw Sedoa
+twb Tawbuid
+twe Teiwa
+twu Termanu
+txa Tombonuo
+txq Tii
+txu Kayapó
+tye Kyanga
+tzh-dialect_bachajon Tzeltal
+tzh-dialect_tenejapa Tzeltal
+tzj-dialect_eastern Tz’utujil
+tzj-dialect_western Tz’utujil
+tzo-dialect_chamula Tzotzil
+tzo-dialect_chenalho Tzotzil
+ubl Bikol, Buhi’non
+ubu Umbu-Ungu
+udm Udmurt
+udu Uduk
+uig-script_arabic Uyghur
+uig-script_cyrillic Uyghur
+ukr Ukrainian
+umb Umbundu
+unr Mundari
+upv Uripiv-Wala-Rano-Atchin
+ura Urarina
+urb Kaapor
+urd-script_arabic Urdu
+urd-script_devanagari Urdu
+urd-script_latin Urdu
+urk Urak Lawoi’
+urt Urat
+ury Orya
+usp Uspanteko
+uzb-script_cyrillic Uzbek
+uzb-script_latin Uzbek
+vag Vagla
+vid Vidunda
+vie Vietnamese
+vif Vili
+vmw Makhuwa
+vmy Mazatec, Ayautla
+vot Vod
+vun Vunjo
+vut Vute
+wal-script_ethiopic Wolaytta
+wal-script_latin Wolaytta
+wap Wapishana
+war Waray-Waray
+waw Waiwai
+way Wayana
+wba Warao
+wlo Wolio
+wlx Wali
+wmw Mwani
+wob Wè Northern
+wol Wolof
+wsg Gondi, Adilabad
+wwa Waama
+xal Kalmyk-Oirat
+xdy Malayic Dayak
+xed Hdi
+xer Xerénte
+xho Xhosa
+xmm Malay, Manado
+xnj Chingoni
+xnr Kangri
+xog Soga
+xon Konkomba
+xrb Karaboro, Eastern
+xsb Sambal
+xsm Kasem
+xsr Sherpa
+xsu Sanumá
+xta Mixtec, Alcozauca
+xtd Mixtec, Diuxi-Tilantongo
+xte Ketengban
+xtm Mixtec, Magdalena Peñasco
+xtn Mixtec, Northern Tlaxiaco
+xua Kurumba, Alu
+xuo Kuo
+yaa Yaminahua
+yad Yagua
+yal Yalunka
+yam Yamba
+yao Yao
+yas Nugunu
+yat Yambeta
+yaz Lokaa
+yba Yala
+ybb Yemba
+ycl Lolopo
+ycn Yucuna
+yea Ravula
+yka Yakan
+yli Yali, Angguruk
+yor Yoruba
+yre Yaouré
+yua Maya, Yucatec
+yue-script_traditional Chinese, Yue
+yuz Yuracare
+yva Yawa
+zaa Zapotec, Sierra de Juárez
+zab Zapotec, Western Tlacolula Valley
+zac Zapotec, Ocotlán
+zad Zapotec, Cajonos
+zae Zapotec, Yareni
+zai Zapotec, Isthmus
+zam Zapotec, Miahuatlán
+zao Zapotec, Ozolotepec
+zaq Zapotec, Aloápam
+zar Zapotec, Rincón
+zas Zapotec, Santo Domingo Albarradas
+zav Zapotec, Yatzachi
+zaw Zapotec, Mitla
+zca Zapotec, Coatecas Altas
+zga Kinga
+zim Mesme
+ziw Zigula
+zlm Malay
+zmz Mbandja
+zne Zande
+zos Zoque, Francisco León
+zpc Zapotec, Choapan
+zpg Zapotec, Guevea de Humboldt
+zpi Zapotec, Santa María Quiegolani
+zpl Zapotec, Lachixío
+zpm Zapotec, Mixtepec
+zpo Zapotec, Amatlán
+zpt Zapotec, San Vicente Coatlán
+zpu Zapotec, Yalálag
+zpz Zapotec, Texmelucan
+ztq Zapotec, Quioquitani-Quierí
+zty Zapotec, Yatee
+zul Zulu
+zyb Zhuang, Yongbei
+zyp Chin, Zyphe
+zza Zaza

data/lid/all_langs.tsv

@@ -0,0 +1,4017 @@
+ara Arabic
+eng English
+cmn Chinese, Mandarin
+spa Spanish
+fra French
+mlg Malagasy
+ful Fulah
+swe Swedish
+por Portuguese
+zlm Malay
+sun Sunda
+tuk Turkmen
+vie Vietnamese
+kor Korean
+hin Hindi
+ben Bengali
+som Somali
+asm Assamese
+swh Swahili
+urd Urdu
+hau Hausa
+ind Indonesian
+tat Tatar
+bod Tibetan, Central
+tel Telugu
+mon Mongolian
+aze Azerbaijani
+rus Russian
+tgl Tagalog
+tur Turkish
+mar Marathi
+amh Amharic
+ron Romanian
+yor Yoruba
+tha Thai
+slv Slovene
+heb Hebrew
+mkd Macedonian
+bel Belarusian
+nya Chichewa
+mal Malayalam
+bul Bulgarian
+hun Hungarian
+hat Haitian Creole
+fas Persian
+hrv Croatian
+cat Catalan
+tam Tamil
+orm Oromo
+kmr Kurdish, Northern
+nld Dutch
+cak Kaqchikel
+afr Afrikaans
+pol Polish
+jav Javanese
+lin Lingala
+cym Welsh
+kik Gikuyu
+nob Norwegian Bokmål
+grn Guarani
+snd Sindhi
+kaz Kazakh
+isl Icelandic
+uzb Uzbek
+bos Bosnian
+mya Burmese
+lat Latin
+deu German, Standard
+npi Nepali
+che Chechen
+yue Chinese, Yue
+kat Georgian
+kan Kannada
+lit Lithuanian
+mam Mam
+sqi Albanian
+hye Armenian
+jpn Japanese
+ell Greek
+crh Crimean Tatar
+lav Latvian
+khm Khmer
+bak Bashkort
+poh Poqomchi’
+quc K’iche’
+pan Punjabi, Eastern
+ixl Ixil
+xog Soga
+ces Czech
+tgk Tajik
+cfm Chin, Falam
+fao Faroese
+guj Gujarati
+aka Akan
+ukr Ukrainian
+glg Galician
+ltz Luxembourgish
+sxn Sangir
+sna Shona
+lao Lao
+mlt Maltese
+sin Sinhala
+lug Ganda
+aiw Aari
+kia Kim
+ayo Ayoreo
+dtp Kadazan Dusun
+cmo Mnong, Central
+nhx Nahuatl, Isthmus-Mecayapan
+gag Gagauz
+tzj Tz’utujil
+tuv Turkana
+acr Achi
+mri Maori
+eus Basque
+pus Pushto
+quy Quechua, Ayacucho
+srp Serbian
+ita Italian
+nno Norwegian Nynorsk
+xsm Kasem
+luo Dholuo
+ory Odia
+gur Farefare
+cac Chuj
+quh Quechua, South Bolivian
+ewe Éwé
+kbp Kabiyè
+saq Samburu
+slk Slovak
+xon Konkomba
+fin Finnish
+mos Mòoré
+bwq Bobo Madaré, Southern
+yao Yao
+hne Chhattisgarhi
+rif Tarifit
+new Newar
+hus Huastec
+dyu Jula
+bre Breton
+guh Guahibo
+bis Bislama
+yid Yiddish
+txa Tombonuo
+mnk Mandinka
+uig Uyghur
+bqc Boko
+dan Danish
+ngl Lomwe
+pse Malay, Central
+bam Bamanankan
+mtg Una
+pmf Pamona
+onb Lingao
+ntm Nateni
+tso Tsonga
+bno Bantoanon
+teo Ateso
+uhn Damal
+ycl Lolopo
+bus Bokobaru
+ttq Tamajaq, Tawallammat
+mcr Menya
+seh Sena
+kru Kurux
+lok Loko
+est Estonian
+tpi Tok Pisin
+zne Zande
+bxk Bukusu
+mzi Mazatec, Ixcatlán
+amf Hamer-Banna
+rel Rendille
+sck Sadri
+lcp Lawa, Western
+gbo Grebo, Northern
+adx Tibetan, Amdo
+tcc Datooga
+cnh Chin, Hakha
+pwg Gapapaiwa
+wlx Wali
+rjs Rajbanshi
+thl Tharu, Dangaura
+xal Kalmyk-Oirat
+dos Dogosé
+lis Lisu
+txu Kayapó
+sxb Suba
+gng Ngangam
+ifa Ifugao, Amganad
+beh Biali
+poe Popoloca, San Juan Atzingo
+dga Dagaare, Southern
+dsh Daasanach
+vmw Makhuwa
+mup Malvi
+lnd Lundayeh
+kbo Keliko
+cwa Kabwa
+rol Romblomanon
+khg Tibetan, Khams
+nko Nkonya
+dgi Dagara, Northern
+kml Kalinga, Tanudan
+nxq Naxi
+acn Achang
+pxm Mixe, Quetzaltepec
+wal Wolaytta
+ctg Chittagonian
+dnw Dani, Western
+pui Puinave
+lew Kaili, Ledo
+bfa Bari
+mqj Mamasa
+rmc Romani, Carpathian
+mhy Ma’anyan
+xsr Sherpa
+gri Ghari
+bfy Bagheli
+kqp Kimré
+frd Fordata
+ayr Aymara, Central
+mip Mixtec, Apasco-Apoala
+nym Nyamwezi
+tzh Tzeltal
+kcg Tyap
+tex Tennet
+lbw Tolaki
+sda Toraja-Sa’dan
+kdt Kuay
+bfo Birifor, Malba
+qxl Quichua, Salasaca Highland
+ttc Tektiteko
+bfz Pahari, Mahasu
+mhx Lhao Vo
+sbp Sangu
+mco Mixe, Coatlán
+mbu Mbula-Bwazza
+mxt Mixtec, Jamiltepec
+nzi Nzema
+suz Sunwar
+hlt Chin, Matu
+tzo Tzotzil
+any Anyin
+gna Kaansa
+sid Sidamo
+alp Alune
+maj Mazatec, Jalapa de Díaz
+zim Mesme
+knj Akateko
+zar Zapotec, Rincón
+mxb Mixtec, Tezoatlán
+bdu Oroko
+bbc Batak Toba
+ddn Dendi
+obo Manobo, Obo
+krs Gbaya
+zaq Zapotec, Aloápam
+ife Ifè
+soy Miyobe
+trs Triqui, Chicahuaxtla
+mbj Nadëb
+tuo Tucano
+atb Zaiwa
+vif Vili
+mim Mixtec, Alacatlatzala
+grc Greek, Ancient
+cek Chin, Eastern Khumi
+kfx Pahari, Kullu
+naw Nawuri
+tgj Tagin
+xed Hdi
+hnn Hanunoo
+had Hatam
+kij Kilivila
+nlc Nalca
+kek Q’eqchi’
+rej Rejang
+fon Fon
+amk Ambai
+kyb Kalinga, Butbut
+dnj Dan
+oku Oku
+gil Kiribati
+mag Magahi
+lln Lele
+pil Yom
+pls Popoloca, San Marcos Tlacoyalco
+box Buamu
+kwf Kwara’ae
+mgd Moru
+xtm Mixtec, Magdalena Peñasco
+ctd Chin, Tedim
+akb Batak Angkola
+nlg Gela
+bmq Bomu
+bmv Bum
+mgo Meta’
+cla Ron
+rug Roviana
+enx Enxet
+mpm Mixtec, Yosondúa
+gof Gofa
+bom Berom
+mbc Macushi
+btx Batak Karo
+did Didinga
+mej Meyah
+bgq Bagri
+maa Mazatec, San Jerónimo Tecóatl
+nmz Nawdm
+mfk Mofu, North
+aeu Akeu
+mqn Moronene
+tob Toba
+hlb Halbi
+nin Ninzo
+kqe Kalagan
+lex Luang
+mkl Mokole
+icr Islander English Creole
+lns Lamnso’
+tlj Talinga-Bwisi
+bzh Buang, Mapos
+bdh Baka
+kle Kulung
+pib Yine
+vut Vute
+btd Batak Dairi
+xmm Malay, Manado
+yka Yakan
+btt Bete-Bendi
+hoc Ho
+yba Yala
+mib Mixtec, Atatlahuca
+kpq Korupun-Sela
+xsb Sambal
+muy Muyang
+zyp Chin, Zyphe
+bbo Konabéré
+krc Karachay-Balkar
+eka Ekajuk
+mcp Makaa
+bqj Bandial
+mcq Ese
+ybb Yemba
+hyw Armenian, Western
+tmc Tumak
+mih Mixtec, Chayuco
+blt Tai Dam
+zpz Zapotec, Texmelucan
+tng Tobanga
+not Nomatsigenga
+pny Pinyin
+nuj Nyole
+bhz Bada
+kvn Kuna, Border
+lje Rampi
+sne Bidayuh, Bau
+ndy Lutos
+ksb Shambala
+nhy Nahuatl, Northern Oaxaca
+kwd Kwaio
+moz Mukulu
+cmr Mro-Khimi
+xuo Kuo
+zpu Zapotec, Yalálag
+avn Avatime
+pap Papiamentu
+pss Kaulong
+akp Siwu
+ted Krumen, Tepo
+rro Waima
+muv Muthuvan
+gau Gadaba, Mudhili
+ake Akawaio
+guq Aché
+lsi Lacid
+cul Kulina
+tna Tacana
+cle Chinantec, Lealao
+iri Rigwe
+flr Fuliiru
+bkd Binukid
+bmr Muinane
+twb Tawbuid
+ikk Ika
+tbl Tboli
+mnw Mon
+asa Asu
+abi Abidji
+yaz Lokaa
+bgw Bhatri
+miy Mixtec, Ayutla
+gai Mbore
+smo Samoan
+cnl Chinantec, Lalana
+far Fataleka
+poi Popoluca, Highland
+tgo Sudest
+gud Dida, Yocoboué
+kak Kalanguya
+gub Guajajára
+yre Yaouré
+cso Chinantec, Sochiapam
+gwr Gwere
+ati Attié
+urt Urat
+mil Mixtec, Peñoles
+ndv Ndut
+rnl Ranglong
+sch Sakachep
+zpc Zapotec, Choapan
+tom Tombulu
+tnt Tontemboan
+atg Ivbie North-Okpela-Arhe
+kdl Tsikimba
+mto Mixe, Totontepec
+bov Tuwuli
+myy Macuna
+ava Avar
+ami Amis
+luc Aringa
+plw Palawano, Brooke’s Point
+cab Garifuna
+sey Paicoca
+zpg Zapotec, Guevea de Humboldt
+xnj Chingoni
+kdc Kutu
+zpt Zapotec, San Vicente Coatlán
+prk Wa, Parauk
+qxr Quichua, Cañar Highland
+nga Ngbaka
+ubl Bikol, Buhi’non
+crs Seychelles French Creole
+cwe Kwere
+pps Popoloca, San Luís Temalacayuca
+bjw Bakwé
+aia Arosi
+taq Tamasheq
+idd Ede Idaca
+ceb Cebuano
+blh Kuwaa
+kfw Naga, Kharam
+gqr Gor
+suc Subanon, Western
+cok Cora, Santa Teresa
+kzf Kaili, Da’a
+myv Erzya
+mge Mango
+tly Talysh
+udm Udmurt
+tmf Toba-Maskoy
+cbi Chachi
+kqr Kimaragang
+yas Nugunu
+nsu Nahuatl, Sierra Negra
+pez Penan, Eastern
+moa Mwan
+dgk Dagba
+tao Yami
+lon Lomwe, Malawi
+kog Kogi
+tlb Tobelo
+azg Amuzgo, San Pedro Amuzgos
+xtd Mixtec, Diuxi-Tilantongo
+bqp Bisã
+kpv Komi-Zyrian
+hwc Hawaii Pidgin
+cpu Ashéninka, Pichis
+yat Yambeta
+kje Kisar
+met Mato
+zmz Mbandja
+ury Orya
+cpb Ashéninka, Ucayali-Yurúa
+bep Behoa
+yea Ravula
+zga Kinga
+asg Cishingini
+kaq Capanahua
+jun Juang
+knb Kalinga, Lubuagan
+kyf Kouya
+rap Rapa Nui
+ess Yupik, Saint Lawrence Island
+stn Owa
+byr Yipma
+sjm Mapun
+mjv Mannan
+rub Gungu
+kjh Khakas
+kmd Kalinga, Majukayang
+dbq Daba
+wap Wapishana
+blx Ayta, Mag-Indi
+kne Kankanaey
+arl Arabela
+abp Ayta, Abellen
+tuf Tunebo, Central
+cgc Kagayanen
+ksr Borong
+ojb Ojibwa, Northwestern
+cbr Kakataibo-Kashibo
+chv Chuvash
+ktj Krumen, Plapo
+omw Tairora, South
+cjo Ashéninka, Pajonal
+mhr Mari, Meadow
+atq Aralle-Tabulahan
+rkt Rangpuri
+ium Iu Mien
+crt Chorote, Iyojwa’ja
+nog Nogai
+snn Siona
+tte Bwanabwana
+tvw Sedoa
+pjt Pitjantjatjara
+nlk Yali, Ninia
+tih Murut, Timugon
+ppk Uma
+lid Nyindrou
+cui Cuiba
+cot Caquinte
+tav Tatuyo
+log Logo
+prt Prai
+boj Anjam
+huu Witoto, Murui
+mqf Momuna
+med Melpa
+snp Siane
+dah Gwahatike
+tnr Ménik
+tbk Tagbanwa, Calamian
+mtj Moskona
+men Mende
+ubu Umbu-Ungu
+agu Awakateko
+kmu Kanite
+trn Trinitario
+zaj Zaramo
+dnt Dani, Mid Grand Valley
+qvh Quechua, Huamalíes-Dos de Mayo Huánuco
+mcd Sharanahua
+urb Kaapor
+wsg Gondi, Adilabad
+war Waray-Waray
+ame Yanesha’
+cof Tsafiki
+bbb Barai
+hap Hupla
+law Lauje
+crq Chorote, Iyo’wujwa
+bor Borôro
+kri Krio
+nhe Nahuatl, Eastern Huasteca
+bjr Binumarien
+xte Ketengban
+eip Lik
+dav Dawida
+mpd Machinere
+mai Maithili
+sil Sisaala, Tumulung
+pis Pijin
+crk Cree, Plains
+kyz Kayabí
+ngu Nahuatl, Guerrero
+guo Guayabero
+mnx Sougb
+nij Ngaju
+qva Quechua, Ambo-Pasco
+lif Limbu
+bvz Bauzi
+awa Awadhi
+kir Kyrgyz
+kin Kinyarwanda
+iba Iban
+niy Ngiti
+nas Naasioi
+knk Kuranko
+gog Gogo
+gvc Wanano
+mdm Mayogo
+pkb Kipfokomo
+sho Shanga
+gbm Garhwali
+dig Chidigo
+bsq Bassa
+tye Kyanga
+gux Gourmanchéma
+yal Yalunka
+zyb Zhuang, Yongbei
+run Rundi
+bky Bokyi
+yan Mayangna
+tbt Tembo
+set Sentani
+oci Occitan
+nyy Nyakyusa-Ngonde
+shn Shan
+bcc Balochi, Southern
+kno Kono
+yaa Yaminahua
+bwu Buli
+bgr Chin, Bawm
+mfz Mabaan
+keo Kakwa
+led Lendu
+kue Kuman
+grt Garo
+sus Susu
+mdy Male
+sah Yakut
+dug Chiduruma
+pkr Kurumba, Attapady
+tir Tigrigna
+suk Sukuma
+san Sanskrit
+kdj Ng’akarimojong
+nyf Kigiryama
+bem Bemba
+hak Chinese, Hakka
+dag Dagbani
+nan Chinese, Min Nan
+kdh Tem
+gum Misak
+hnj Hmong Njua
+aha Ahanta
+lsm Saamya-Gwe
+nyn Nyankore
+lam Lamba
+tgw Sénoufo, Tagwana
+kde Makonde
+lhu Lahu
+wme Wambule
+guc Wayuu
+mur Murle
+kam Kamba
+bru Bru, Eastern
+nsk Naskapi
+guk Gumuz
+cas Tsimané
+nnw Nuni, Southern
+jow Jowulu
+bvc Baelelea
+gjn Gonja
+cko Anufo
+rim Nyaturu
+mfi Wandala
+thf Thangmi
+trq Triqui, San Martín Itunyoso
+bmu Somba-Siawari
+ade Adele
+rmy Romani, Vlax
+nim Nilamba
+mbb Manobo, Western Bukidnon
+mxv Mixtec, Metlatónoc
+ses Songhay, Koyraboro Senni
+dyo Jola-Fonyi
+taj Tamang, Eastern
+mnb Muna
+sbd Samo, Southern
+hui Huli
+esi Inupiatun, North Alaskan
+wba Warao
+kqn Kaonde
+spy Sabaot
+raw Rawang
+kbr Kafa
+tem Themne
+bst Basketo
+oss Ossetic
+omi Omi
+qul Quechua, North Bolivian
+car Carib
+kff Koya
+ptu Bambam
+mev Maan
+mgh Makhuwa-Meetto
+cly Chatino, Eastern Highland
+mpx Misima-Panaeati
+kus Kusaal
+mwq Chin, Müün
+khq Songhay, Koyra Chiini
+nia Nias
+urk Urak Lawoi’
+spp Sénoufo, Supyire
+dzo Dzongkha
+sgb Ayta, Mag-antsi
+kma Konni
+iou Tuma-Irumu
+lef Lelemi
+nst Naga, Tangshang
+udg Muduga
+vag Vagla
+kum Kumyk
+maw Mampruli
+quz Quechua, Cusco
+kaa Karakalpak
+mpg Marba
+yva Yawa
+bgc Haryanvi
+bim Bimoba
+fij Fijian
+bud Ntcham
+ceg Chamacoco
+tpm Tampulma
+mrj Mari, Hill
+nus Nuer
+sba Ngambay
+lom Loma
+bib Bisa
+twu Termanu
+acd Gikyode
+mak Makasar
+cni Asháninka
+pbb Nasa
+qvm Quechua, Margos-Yarowilca-Lauricocha
+zab Zapotec, Western Tlacolula Valley
+csk Jola-Kasa
+gxx Wè Southern
+bgt Bughotu
+yuz Yuracare
+emp Emberá, Northern
+mzj Manya
+mfq Moba
+guw Gun
+kac Jingpho
+ilo Ilocano
+qvo Quichua, Napo
+las Lama
+ctu Chol
+cdj Churahi
+yam Yamba
+dip Dinka, Northeastern
+kfy Kumaoni
+sig Paasaal
+srx Sirmauri
+mie Mixtec, Ocotepec
+tca Ticuna
+cap Chipaya
+nav Navajo
+mca Maka
+pce Palaung, Ruching
+upv Uripiv-Wala-Rano-Atchin
+bgd Bareli, Rathwi
+blz Balantak
+dik Dinka, Southwestern
+gbi Galela
+dgo Dogri
+nnb Nande
+cax Chiquitano
+myb Mbay
+txq Tii
+dhi Dhimal
+mad Madura
+shk Shilluk
+ktb Kambaata
+quw Quichua, Tena Lowland
+rav Sampang
+sag Sango
+nyu Nyungwe
+ljp Lampung Api
+mzm Mumuye
+stb Subanen, Northern
+pab Parecís
+mzw Deg
+mhi Ma’di
+gor Gorontalo
+agd Agarabi
+gnd Zulgo-Gemzek
+xnr Kangri
+mor Moro
+kyu Kayah, Western
+ese Ese Ejja
+myk Sénoufo, Mamara
+zaw Zapotec, Mitla
+cme Cerma
+aaz Amarasi
+cnt Chinantec, Tepetotutla
+zpo Zapotec, Amatlán
+anv Denya
+ach Acholi
+mwv Mentawai
+kfb Kolami, Northwestern
+otn Otomi, Tenango
+kbq Kamano
+kss Kisi, Southern
+jiv Shuar
+dop Lukpa
+nhw Nahuatl, Western Huasteca
+ahk Akha
+tbz Ditammari
+zas Zapotec, Santo Domingo Albarradas
+huv Huave, San Mateo del Mar
+xtn Mixtec, Northern Tlaxiaco
+bex Jur Modo
+crn Cora, El Nayar
+cuk Kuna, San Blas
+gbk Gaddi
+toi Tonga
+key Kupia
+ifb Ifugao, Batad
+ztq Zapotec, Quioquitani-Quierí
+nag Nagamese
+toc Totonac, Coyutla
+ken Kenyang
+agr Awajún
+bfd Bafut
+kyq Kenga
+ker Kera
+ntr Delo
+usp Uspanteko
+alz Alur
+mas Maasai
+lme Pévé
+nhu Noone
+dwr Dawro
+ksp Kabba
+ncu Chumburung
+min Minangkabau
+wol Wolof
+hif Hindi, Fiji
+tll Tetela
+bba Baatonum
+cco Chinantec, Comaltepec
+tbc Takia
+lia Limba, West-Central
+mgq Malila
+mnf Mundani
+hil Hiligaynon
+kyc Kyaka
+ozm Koonzime
+gyr Guarayu
+pcm Pidgin, Nigerian
+sml Sama, Central
+npl Nahuatl, Southeastern Puebla
+tby Tabaru
+lem Nomaande
+udu Uduk
+xsu Sanumá
+soq Kanasi
+tik Tikar
+ibg Ibanag
+zpl Zapotec, Lachixío
+sbl Sambal, Botolan
+itv Itawit
+noa Woun Meu
+ace Aceh
+ign Ignaciano
+shp Shipibo-Conibo
+jbu Jukun Takum
+kub Kutep
+knf Mankanya
+mvp Duri
+jac Jakalteko
+wwa Waama
+biv Birifor, Southern
+kkj Kako
+ter Terêna
+pbi Parkwa
+csy Chin, Siyin
+xrb Karaboro, Eastern
+mxq Mixe, Juquila
+mfh Matal
+bht Bhattiyali
+fal Fali, South
+adj Adioukrou
+mcu Mambila, Cameroon
+otq Otomi, Querétaro
+bpr Blaan, Koronadal
+miq Mískito
+tee Tepehua, Huehuetla
+mrw Maranao
+nfr Nafaanra
+izr Izere
+bzi Bisu
+sas Sasak
+cou Wamey
+cbt Shawi
+lwo Luwo
+ban Bali
+kab Amazigh
+cbs Kashinawa
+prf Paranan
+nhi Nahuatl, Zacatlán-Ahuacatlán-Tepetzintla
+dyi Sénoufo, Djimini
+cnw Chin, Ngawn
+zaa Zapotec, Sierra de Juárez
+mfe Morisyen
+mio Mixtec, Pinotepa Nacional
+kjb Q’anjob’al
+myx Masaaba
+con Cofán
+bkv Bekwarra
+sur Mwaghavul
+eza Ezaa
+qxn Quechua, Northern Conchucos Ancash
+lgg Lugbara
+cya Chatino, Nopala
+zao Zapotec, Ozolotepec
+kez Kukele
+sja Epena
+bdq Bahnar
+acf Lesser Antillean French Creole
+ruf Luguru
+cce Chopi
+old Mochi
+acu Achuar-Shiwiar
+jmc Machame
+xpe Kpelle, Liberia
+alj Alangan
+awb Awa
+srn Sranan Tongo
+zad Zapotec, Cajonos
+lob Lobi
+tsz Purepecha
+ote Otomi, Mezquital
+bcl Bikol, Central
+mbt Manobo, Matigsalug
+yua Maya, Yucatec
+sgw Sebat Bet Gurage
+tue Tuyuca
+kao Xaasongaxango
+mjl Mandeali
+maz Mazahua, Central
+miz Mixtec, Coatzospan
+qvw Quechua, Huaylla Wanca
+cpa Chinantec, Palantla
+kxc Konso
+bss Akoose
+laj Lango
+nyo Nyoro
+ndp Kebu
+hag Hanga
+lip Sekpele
+agn Agutaynen
+mfy Mayo
+nod Thai, Northern
+zos Zoque, Francisco León
+gde Gude
+qub Quechua, Huallaga
+tri Trió
+way Wayana
+umb Umbundu
+gwi Gwich’in
+qwh Quechua, Huaylas Ancash
+bsc Oniyan
+qvn Quechua, North Junín
+ncj Nahuatl, Northern Puebla
+tnk Kwamera
+mit Mixtec, Southern Puebla
+irk Iraqw
+djk Aukan
+vun Vunjo
+rai Ramoaaina
+mda Mada
+gym Ngäbere
+wob Wè Northern
+pam Kapampangan
+mop Maya, Mopán
+tpp Tepehua, Pisaflores
+mzk Mambila, Nigeria
+hig Kamwe
+tap Taabwa
+hto Witoto, Minika
+pww Karen, Pwo Northern
+kxm Khmer, Northern
+pbc Patamona
+ifu Ifugao, Mayoyao
+heh Hehe
+bnp Bola
+nwb Nyabwa
+pko Pökoot
+jam Jamaican English Creole
+gej Gen
+sld Sissala
+iqw Ikwo
+pae Pagibete
+tac Tarahumara, Western
+zai Zapotec, Isthmus
+alt Altai, Southern
+snw Selee
+ann Obolo
+lee Lyélé
+bao Waimaha
+klv Maskelynes
+izz Izii
+pag Pangasinan
+thk Kitharaka
+hay Haya
+mog Mongondow
+krj Kinaray-a
+klu Klao
+apb Sa’a
+gmv Gamo
+ycn Yucuna
+kqy Koorete
+msy Aruamu
+qvs Quechua, San Martín
+ood Tohono O’odham
+cbc Carapana
+stp Tepehuan, Southeastern
+bts Batak Simalungun
+enb Markweeta
+bcw Bana
+muh Mündü
+adh Jopadhola
+gkn Gokana
+tgp Tangoa
+ziw Zigula
+kpz Kupsapiiny
+poy Pogolo
+daa Dangaléat
+tnn Tanna, North
+shi Tachelhit
+guu Yanomamö
+kdi Kumam
+ata Pele-Ata
+bav Vengo
+neb Toura
+mif Mofu-Gudur
+mbh Mangseng
+srm Saramaccan
+vid Vidunda
+vmy Mazatec, Ayautla
+nnq Ngindo
+dts Dogon, Toro So
+ilb Ila
+ngp Ngulu
+tpt Tepehua, Tlachichilco
+kki Kagulu
+gvl Gulay
+chz Chinantec, Ozumacín
+ndj Ndamba
+toh Tonga
+zae Zapotec, Yareni
+caa Ch’orti’
+pau Palauan
+zpi Zapotec, Santa María Quiegolani
+cjp Cabécar
+bng Benga
+bjv Bedjond
+cuc Chinantec, Usila
+krl Karelian
+wmw Mwani
+nch Nahuatl, Central Huasteca
+nse Nsenga
+ndz Ndogo
+meq Merey
+mah Marshallese
+gso Gbaya, Southwest
+kwi Awa-Cuaiquer
+qve Quechua, Eastern Apurímac
+mza Mixtec, Santa María Zacatepec
+rng Ronga
+azz Nahuatl, Highland Puebla
+hns Hindustani, Sarnami
+npy Napu
+bps Blaan, Sarangani
+mqb Mbuko
+ura Urarina
+zty Zapotec, Yatee
+inb Inga
+cwt Kuwaataay
+yli Yali, Angguruk
+pad Paumarí
+mox Molima
+zpm Zapotec, Mixtepec
+tos Totonac, Highland
+bzj Belize English Creole
+apr Arop-Lokep
+ifk Ifugao, Tuwali
+nca Iyo
+boa Bora
+rmo Romani, Sinte
+jic Tol
+ded Dedua
+waw Waiwai
+saj Sahu
+lnl Banda, South Central
+pir Piratapuyo
+quf Quechua, Lambayeque
+sri Siriano
+kdn Kunda
+cbv Cacua
+lac Lacandon
+mpp Migabac
+gam Kandawo
+qvc Quechua, Cajamarca
+qvz Quichua, Northern Pastaza
+qxh Quechua, Panao
+lai Lambya
+hub Wampís
+jvn Javanese, Suriname
+coe Koreguaje
+ify Kallahan, Keley-i
+nab Nambikuára, Southern
+mir Mixe, Isthmus
+apf Agta, Pahanan
+des Desano
+lww Lewo
+cbu Kandozi-Chapra
+tfr Teribe
+beq Beembe
+nbw Ngbandi, Southern
+loq Lobala
+tbg Tairora, North
+avu Avokaya
+mcb Matsigenka
+bto Bikol, Rinconada
+mnh Mono
+lgl Wala
+yad Yagua
+qxo Quechua, Southern Conchucos
+hno Hindko, Northern
+bxg Bangala
+pao Paiute, Northern
+ibo Igbo
+jnj Yemsa
+sgj Surgujia
+ldi Laari
+sab Buglere
+bci Baoulé
+bxh Buhutu
+haw Hawaiian
+tnc Tanimuca-Letuama
+mfx Melo
+tyv Tuvan
+neq Mixe, North Central
+wbi Vwanji
+bcq Bench
+ksw Karen, S’gaw
+guz Ekegusii
+mkw Kituba
+ore Maijuna
+ige Igede
+bjz Baruga
+zca Zapotec, Coatecas Altas
+mer Kimîîru
+aui Anuki
+arn Mapudungun
+zul Zulu
+kxf Kawyaw
+alw Alaba-K’abeena
+xho Xhosa
+loz Lozi
+mww Hmong Daw
+mey Hassaniyya
+ijc Izon
+mwt Moken
+bza Bandi
+lun Lunda
+kby Kanuri, Manga
+pov Guinea-Bissau Creole
+bdg Bonggi
+ipi Ipili
+sfw Esahie
+knc Kanuri, Yerwa
+syl Sylheti
+bho Bhojpuri
+tum Tumbuka
+tdy Tadyawan
+nso Sotho, Northern
+lbj Ladakhi
+ckb Kurdish, Central
+ndc Ndau
+bwr Bura-Pabir
+pci Duruwa
+dje Zarma
+bax Bamun
+top Totonac, Papantla
+gkp Kpelle, Guinea
+lub Luba-Katanga
+qug Quichua, Chimborazo Highland
+lus Mizo
+csh Chin, Asho
+gvr Gurung
+tew Tewa
+cag Nivaclé
+bev Bété, Daloa
+ggu Gban
+vai Vai
+tiv Tiv
+dgr Tlicho
+epo Esperanto
+srr Serer-Sine
+elm Eleme
+maf Mafa
+abk Abkhaz
+ijn Kalabari
+lua Luba-Kasai
+kck Kalanga
+ngb Ngbandi, Northern
+zpq Zapotec, Zoogocho
+etu Ejagham
+gvs Gumawana
+bft Balti
+tzm Tamazight, Central Atlas
+ida Luidakho-Luisukha-Lutirichi
+enl Enlhet
+ada Dangme
+nzb Njebi
+xdy Malayic Dayak
+aca Achagua
+ktu Kituba
+ebu Kiembu
+pdt Plautdietsch
+gaa Ga
+swk Sena, Malawi
+awn Awngi
+okr Kirike
+kvj Psikye
+xkl Kenyah, Mainstream
+knp Kwanja
+krw Krahn, Western
+mzl Mixe, Mazatlán
+ndi Samba Leko
+mug Musgu
+soe Songomeno
+sea Semai
+kfc Konda-Dora
+lol Mongo-Nkundu
+tsc Tswa
+idu Idoma
+mni Meitei
+trc Triqui, Copala
+mgr Mambwe-Lungu
+mcn Masana
+lrc Luri, Northern
+kfi Kurumba, Kannada
+bzw Basa
+mzz Maiadomu
+mrt Marghi Central
+rml Romani, Baltic
+rhg Rohingya
+urh Urhobo
+lag Langi
+its Isekiri
+ego Eggon
+gle Irish
+ubr Ubir
+hdy Hadiyya
+jen Dza
+sru Suruí
+ngc Ngombe
+lmp Limbum
+isn Isanzu
+kqs Kissi, Northern
+kpm Koho
+nup Nupe-Nupe-Tako
+mwm Sar
+kng Koongo
+nnc Nancere
+bkm Kom
+tui Tupuri
+ogo Khana
+lic Hlai
+mkn Malay, Kupang
+wed Wedau
+ald Alladian
+ksf Bafia
+dur Dii
+jaa Jamamadí
+kmb Kimbundu
+mua Mundang
+cje Chru
+igb Ebira
+gya Gbaya, Northwest
+skr Saraiki
+dow Doyayo
+dww Dawawa
+iso Isoko
+giz Giziga
+bum Bulu
+zza Zaza
+mfa Malay, Pattani
+snf Noon
+mgw Matumbi
+bin Edo
+vmk Makhuwa-Shirima
+dua Duala
+kea Kabuverdianu
+sef Sénoufo, Cebaara
+kaj Jju
+kqf Kakabai
+ayz Mai Brat
+ksz Kodaku
+ncl Nahuatl, Michoacán
+bzd Bribri
+ssn Waata
+mro Mru
+bhi Bhilali
+wes Pidgin, Cameroon
+adi Adi
+efi Efik
+ena Apal
+nde Ndebele
+ast Asturian
+mhw Mbukushu
+bbj Ghomálá’
+geb Kire
+igl Igala
+aoi Anindilyakwa
+rao Rao
+nnh Ngiemboon
+byv Medumba
+sat Santhali
+dzg Dazaga
+gnn Gumatj
+bhb Bhili
+swp Suau
+sgc Kipsigis
+wim Wik-Mungkan
+viv Iduna
+ady Adyghe
+krr Krung
+fan Fang
+coh Chichonyi-Chidzihana-Chikauma
+nbq Nggem
+gvo Gavião do Jiparaná
+glk Gilaki
+acz Acheron
+mwf Murrinh-Patha
+wbp Warlpiri
+tod Toma
+unr Mundari
+khe Korowai
+ntj Ngaanyatjarra
+wnc Wantoat
+suj Shubi
+emk Maninkakan, Eastern
+kel Kela
+dks Dinka, Southeastern
+zav Zapotec, Yatzachi
+jra Jarai
+dhg Dhangu-Djangu
+wlo Wolio
+bmk Ghayavi
+lgr Lengo
+njz Nyishi
+lue Luvale
+mhu Digaro-Mishmi
+tsn Setswana
+beo Bedamuni
+lgm Lega-Mwenga
+haq Ha
+trp Kok Borok
+tdh Thulung
+tuy Tugen
+lzz Laz
+gvj Guajá
+gom Konkani, Goan
+kjl Kham, Western Parbate
+tke Takwane
+mpj Martu Wangka
+ven Venda
+xer Xerénte
+nyd Olunyole
+byd Benyadu’
+snc Sinaugoro
+sdr Sadri, Oraon
+toq Toposa
+wod Wolani
+nhr Naro
+tvs Taveta
+ble Balanta-Kentohe
+bcp Bali
+rag Lulogooli
+jmx Mixtec, Western Juxtlahuaca
+bvd Baeggu
+bvu Malay, Bukit
+dbj Ida’an
+her Herero
+mwc Are
+sou Thai, Southern
+ktz Juǀ’hoansi
+rmn Romani, Balkan
+qxu Quechua, Arequipa-La Unión
+nmn !Xóõ
+haj Hajong
+bee Byangsi
+wbf Wara
+sot Sotho, Southern
+fmu Muria, Far Western
+swb Comorian, Maore
+dde Doondo
+mve Marwari
+mlk Kiwilwana
+mjt Sauria Paharia
+bjg Bidyogo
+jmd Yamdena
+mwn Nyamwanga
+yml Iamalele
+kha Khasi
+mzp Movima
+tvk Ambrym, Southeast
+tkr Tsakhur
+dim Dime
+mix Mixtec, Mixtepec
+tbo Tawala
+lma Limba, East
+pln Palenquero
+koe Suri, Kacipo-Bale
+glv Manx
+kjg Khmu
+wof Wolof, Gambian
+kjc Konjo, Coastal
+xuu Khwedam
+brv Bru, Western
+aoz Uab Meto
+evn Evenki
+tsb Tsamai
+djr Djambarrpuyngu
+mch Maquiritari
+kgk Kaiwá
+klr Khaling
+gno Gondi, Northern
+nuy Nunggubuyu
+srq Sirionó
+sep Sénoufo, Sìcìté
+oki Okiek
+trd Turi
+msc Maninka, Sankaran
+twm Monpa, Tawang
+rki Rakhine
+mfv Mandjak
+mhs Buru
+mjx Mahali
+ggw Gogodala
+nfa Dhao
+mym Me’en
+hvn Hawu
+nuz Nahuatl, Tlamacazapa
+are Arrarnta, Western
+lbm Lodhi
+hni Hani
+chf Chontal, Tabasco
+mtd Mualang
+div Maldivian
+the Tharu, Central
+rgs Roglai, Southern
+nys Nyungar
+tpe Tippera
+eyo Keiyo
+ghr Ghera
+kls Kalasha
+lrm Olumarama
+pmy Malay, Papuan
+lbx Lawangan
+akh Angal Heneng
+kpc Curripaco
+sco Scots
+lwg Oluwanga
+kay Kamayurá
+zac Zapotec, Ocotlán
+ccp Chakma
+pof Poke
+seg Segeju
+nos Nisu, Eastern
+abt Ambulas
+llc Lele
+sbe Saliba
+khz Keapara
+yup Yukpa
+khw Khowar
+bjn Banjar
+kyg Keyagana
+tab Tabasaran
+wci Gbe, Waci
+llg Lole
+lig Ligbi
+tcz Chin, Thado
+tog Tonga
+bqi Bakhtiâri
+psa Awyu, Asue
+knx Kendayan
+wat Kaninuwa
+xem Kembayan
+suv Puroik
+hix Hixkaryána
+bmf Bom-Kim
+bkx Baikeno
+imo Imbongu
+cjs Shor
+cto Embera Catío
+nyk Nyaneka
+tet Tetun
+slu Selaru
+xmc Makhuwa-Marrevone
+knu Kono
+rgu Rikou
+bgz Banggai
+zam Zapotec, Miahuatlán
+xdn 
+iru Irula
+mbp Malayo
+ymm Maay
+kuj Kuria
+bfg Kayan, Busang
+thq Tharu, Mid-Eastern
+otd Ot Danum
+tnv Tangchangya
+esg Gondi, Aheri
+ajg Aja
+dwy Dhuwaya
+yrl Nhengatu
+kud ’Auhelawa
+mau Mazatec, Huautla
+loe Saluan
+kiw Kiwai, Northeast
+zin Zinza
+bbr Girawa
+srb Sora
+gup Gunwinggu
+pht Phu Thai
+ztg Zapotec, Xanaguía
+tpa Taupota
+blr Blang
+awi Aekyom
+pgg Pangwali
+snk Soninke
+nni Nuaulu, North
+hts Hadza
+scg Sanggau
+xdo Kwandu
+adq Adangbe
+cnk Chin, Khumi
+nza Mbembe, Tigon
+agg Angor
+ina Interlingua (International Auxiliary Language Association)
+maq Mazatec, Chiquihuitlán
+blo Anii
+ctp Chatino, Western Highland
+lbf Tinani
+xta Mixtec, Alcozauca
+tix Tiwa, Southern
+mee Mengen
+dnn Dzùùngoo
+kap Bezhta
+ssy Saho
+yon Yongkom
+tlr Talise
+duc Duna
+tro Naga, Tarao
+tth Ta’oih, Upper
+kpo Ikposo
+nuf Nusu
+pbo Papel
+lla Lala-Roba
+mki Dhatki
+ckt Chukchi
+pri Paicî
+pnb Punjabi, Western
+rah Rabha
+fli Fali Muchella
+eto Eton
+beu Blagar
+xsq Makhuwa-Saka
+bhw Biak
+atd Manobo, Ata
+zpv Zapotec, Chichicapan
+sza Semelai
+bob Aweer
+afz Obokuitai
+mui Musi
+tkt Tharu, Kathariya
+phr Pahari-Potwari
+bha Bharia
+tdt Tetun Dili
+ton Tongan
+nwi Tanna, Southwest
+olu Kuvale
+mxx Mahou
+uki Kui
+mgp Magar, Eastern
+zgb Zhuang, Guibei
+bxr Buriat, Russia
+tsj Tshangla
+gwn Gwandara
+bon Bine
+enq Enga
+qxp Quechua, Puno
+bji Burji
+onr One, Northern
+xky Uma’ Lasan
+awu Awyu, Central
+kvo Dobel
+xav Xavánte
+yiu Awu
+sdq Semandang
+pdu Kayan
+vaa Vaagri Booli
+shr Shi
+kvw Wersing
+mvv Murut, Tahol
+blb Bilua
+ckh Chak
+kei Kei
+jml Jumli
+knl Keninjal
+tpr Tuparí
+pwo Karen, Pwo Western
+dgc Agta, Casiguran Dumagat
+bug Bugis
+age Angal
+kmw Komo
+sei Seri
+cbn Nyahkur
+ria Riang
+asy Asmat, Yaosakor
+nes Kinnauri, Bhoti
+mrr Maria
+oyb Oy
+vah Varhadi-Nagpuri
+gnk ǁGana
+gah Alekano
+ghe Ghale, Southern
+aoj Mufian
+kps Tehit
+tpx Me’phaa, Acatepec
+jab Hyam
+vaj Northwestern !Kung
+sie Simaa
+pcf Paliyan
+itl Itelmen
+gld Nanai
+hmd Miao, Large Flowery
+skx Seko Padang
+yoy Yoy
+dhw Danuwar
+sbu Stod Bhoti
+bun Sherbro
+khb Lü
+leu Kara
+kas Kashmiri
+hii Hinduri
+djo Jangkang
+krn Sapo
+bap Bantawa
+iii Nuosu
+row Dela-Oenale
+brx Boro
+lir Liberian English
+apz Safeyoka
+ssw Swati
+kib Koalib
+bmb Bembe
+cao Chácobo
+nbe Naga, Konyak
+jna Jangshung
+kca Khanty
+zyn Zhuang, Yongnan
+kpy Koryak
+peg Pengo
+tnl Lenakel
+nti Natioro
+gaj Gadsup
+lep Lepcha
+mxn Moi
+dry Darai
+kmc Dong, Southern
+kup Kunimaipa
+tqo Toaripi
+kqb Kovai
+ksd Kuanua
+hea Miao, Northern Qiandong
+pcc Bouyei
+dre Dolpo
+mxj Miju-Mishmi
+lyn Luyana
+kxv Kuvi
+cns Asmat, Central
+aix Aighon
+rwr Marwari
+anu Anuak
+aso Dano
+ino Inoke-Yate
+ncm Nambo
+kfq Korku
+dhn Dangi
+nii Nii
+bzf Boikin
+srl Isirawa
+bpe Bauni
+ong Olo
+mho Mashi
+sdo Bidayuh Serian
+kfv Kurmukar
+cch Atsam
+agx Aghul
+ewo Ewondo
+dta Daur
+mlu To’abaita
+zik Zimakani
+yom Yombe
+lae Pattani
+wbr Wagdi
+dar Dargwa
+mrm Mwerlap
+hmt Hamtai
+vay Wayu
+dib Dinka, South Central
+cdm Chepang
+ola Walungge
+yiz Azhe
+lri Olumarachi
+xmz Mori Bawah
+tpj Ñandeva
+kgp Kaingang
+bcf Bamu
+wib Toussian, Southern
+mji Kim Mun
+fwe Fwe
+apw Apache, Western
+xri Krikati-Timbira
+thr Tharu, Rana
+afe Utugwang-Irungene-Afrike
+gea Geruma
+gwj ǀGwi
+kai Karekare
+sgp Singpho
+ahl Igo
+pav Pakaásnovos
+zzj Zhuang, Zuojiang
+sip Sikkimese
+ybi Yamphu
+cli Chakali
+xtl Mixtec, Tijaltepec
+cro Crow
+pmi Pumi, Northern
+nmi Nyam
+kcl Kala
+ish Esan
+rab Chamling
+kvf Kabalai
+kwv Kaba Naa, Sara
+bwi Baniwa
+mrd Magar, Western
+kfk Kinnauri
+cfa Dikaka
+pex Petats
+aly Alyawarr
+lot Otuho
+twe Teiwa
+ygr Yagaria
+afu Awutu
+gol Gola
+dhd Dhundari
+bku Buhid
+ppt Pa
+ulu Uma’ Lung
+syw Syuba
+ekg Ekari
+boq Bogaya
+tsx Mubami
+stt Stieng, Budeh
+kwl Kofyar
+bzy Abanglekuo
+mjc Mixtec, San Juan Colorado
+tnp Whitesands
+njb Naga, Nocte
+mle Manambu
+ram Canela
+bas Basaa
+kjp Karen, Pwo Eastern
+shj Shatt
+hut Humla
+pud Punan Aput
+att Atta, Pamplona
+wbm Wa, Vo
+xuj Kurumba, Jennu
+bhj Bahing
+dhm Dhimba
+les Lese
+amn Amanab
+ass Ipulo
+kge Komering
+bwx Bunu, Bu-Nao
+onp Sartang
+nmo Naga, Moyon
+gju Gujari
+haz Hazaragi
+snx Sam
+bfb Bareli, Pauri
+kyo Klon
+tdf Talieng
+mgm Mambae
+swv Shekhawati
+blk Pa’o
+kqm Khisa
+ikx Ik
+yig Nasu, Wusa
+twh Tai Dón
+tjg Tunjung
+kpb Kurumba, Mullu
+kzs Sugut Dusun
+szb Ngalum
+ysn Sani
+bzz Evant
+nbu Naga, Rongmei
+cgk Chocangacakha
+kbd Kabardian
+cua Cua
+ntp Tepehuan, Northern
+zpj Zapotec, Quiavicuzas
+aii Assyrian Neo-Aramaic
+kpr Korafe-Yegha
+tpu Tampuan
+mfc Mba
+xra Krahô
+aai Miniafia Oyan
+shg Shua
+brg Baure
+tsg Tausug
+giw Duoluo
+myl Moma
+mks Mixtec, Silacayoapan
+say Saya
+goj Gowlan
+ywq Yi, Wuding-Luquan
+tsr Akei
+niq Nandi
+mtr Mewari
+lml Hano
+wtm Mewati
+mde Maba
+cik Kinnauri, Chitkuli
+dwz Dewas Rai
+uar Tairuma
+ian Iatmul
+lar Larteh
+ttr Tera
+dby Dibiyaso
+pah Tenharim
+wlv Bermejo Wichí
+mpr Vangunu
+uth ut-Hun
+krv Kavet
+mrg Mising
+grv Grebo, Central
+bpx Bareli, Palya
+dob Dobu
+knv Tabo
+scp Hyolmo
+shy Tachawit
+lbe Lak
+sya Siang
+loy Lhowa
+cux Cuicatec, Tepeuxila
+ybh Yakkha
+sso Essono
+ztp Zapotec, Loxicha
+jul Jirel
+kgq Kamoro
+dao Chin, Daai
+wad Wamesa
+mnz Moni
+kbc Kadiwéu
+agw Kahua
+wmt Walmajarri
+bco Kaluli
+pkh Pangkhua
+meu Motu
+gjk Koli, Kachi
+uss us-Saare
+raa Dungmali
+nkb Naga, Khoibu
+aau Abau
+bde Bade
+mzr Marubo
+sax Sa
+txo Toto
+mte Mono
+sdp Sherdukpen
+hmo Motu, Hiri
+gdb Gadaba, Pottangi Ollar
+tic Tira
+mdk Mangbutu
+baa Babatana
+sjp Surjapuri
+kun Kunama
+kbl Kanembu
+mql Mbelime
+qud Quichua, Calderón Highland
+lpo Lipo
+arr Karo
+kty Kango
+klw Tado
+mke Mawchi
+nfu Mfumte
+soi Sonha
+tar Tarahumara, Central
+xub Kurumba, Betta
+klz Kabola
+lra Bakati’, Rara
+mxu Mada
+kwx Khirwar
+mdr Mandar
+hoe Horom
+lsr Aruop
+mbz Mixtec, Amoltepec
+lbq Wampar
+mdd Mbum
+plj Polci
+all Allar
+kjo Kinnauri, Pahari
+xmt Matbat
+kft Kanjari
+mcf Matses
+tbf Mandara
+sif Siamou
+tio Teop
+tcy Tulu
+lnu Longuda
+ica Ede Ica
+bpp Kaure
+juk Wapan
+shb Ninam
+grj Grebo, Southern
+bec Iceve-Maci
+mvg Mixtec, Yucuañe
+cnb Chin, Uppu
+skj Seke
+noe Nimadi
+tba Aikanã
+sly Selayar
+dot Dass
+sfm Miao, Small Flowery
+yss Yessan-Mayo
+blw Balangao
+slr Salar
+soa Thai Song
+bla Blackfoot
+tan Tangale
+bns Bundeli
+xtc Katcha-Kadugli-Miri
+nmf Naga, Tangkhul
+grd Guruntum-Mbaaru
+amr Amarakaeri
+puu Punu
+mlm Mulam
+lec Leco
+bcs Hohumono
+byn Bilen
+ott Otomi, Temoaya
+arv Arbore
+xkk Kachok
+mjg Tu
+pnq Pana
+asc Asmat, Casuarina Coast
+aks Akaselem
+mmg Ambrym, North
+tld Talaud
+bkq Bakairí
+ort Oriya, Adivasi
+kxz Kerewo
+kwj Kwanga
+cub Cubeo
+eja Jola-Felupe
+wbl Wakhi
+uri Urim
+zua Zeem
+kjd Kiwai, Southern
+ruk Kuce
+lbk Bontok, Central
+bfw Bondo
+jao Yanyuwa
+hca Andaman Hindi Creole
+ssx Samberigi
+ldl Kaan
+byx Qaqet
+nku Kulango, Bouna
+gec Grebo, Gboloo
+zlj Zhuang, Liujiang
+bge Bauria
+btu Batu
+nlx Nahali
+hmr Hmar
+tcu Tarahumara, Southeastern
+lax Tiwa
+lhm Lhomi
+kdp Nikyob-Nindem
+tes Tengger
+mdb Morigi
+msi Malay, Sabah
+rog Roglai, Northern
+jda Jad
+zpa Zapotec, Lachiguiri
+poc Poqomam
+mgu Magi
+nnu Dwang
+kui Kuikúro-Kalapálo
+llp Efate, North
+kxj Kulfa
+mjz Majhi
+jms Mashi
+nto Ntomba
+hsn Chinese, Xiang
+bhu Bhunjia
+nfd Ahwai
+ksg Kusaghe
+kzr Karang
+lyg Lyngngam
+prp Parsi
+lle Lele
+kex Kukna
+brh Brahui
+bkk Brokskat
+wuu Chinese, Wu
+gry Grebo, Barclayville
+bgp Balochi, Eastern
+pai Pye
+cta Chatino, Tataltepec
+cog Chong
+oro Orokolo
+pug Phuie
+swi Sui
+inj Inga, Jungle
+wmo Wom
+kcv Kete
+cna Changthang
+xkf Khengkha
+jer Jere
+bca Bai, Central
+kua Oshiwambo
+roh Romansh
+mxe Mele-Fila
+jmn Naga, Makuri
+dus Dumi
+ssk Sunam
+bqg Bago-Kusuntu
+pwr Powari
+jbj Arandai
+yet Yetfa
+lhi Lahu Shi
+aar Afar
+ksu Khamyang
+mxy Mixtec, Southeastern Nochixtlán
+tcn Tichurong
+lmx Laimbue
+xua Kurumba, Alu
+khr Kharia
+zyj Zhuang, Youjiang
+mng Mnong, Eastern
+roo Rotokas
+anr Andh
+mdv Mixtec, Santa Lucía Monteverde
+msm Manobo, Agusan
+nbl Ndebele
+cin Cinta Larga
+sjl Miji
+saw Sawi
+xkz Kurtokha
+npb Nupbikha
+cnc Côông
+muk Mugom
+foi Foi
+sqq Sou
+tdd Tai Nüa
+kil Kariya
+bma Lame
+dad Marik
+bix Bijori
+nao Naaba
+pwb Panawa
+bhx Bhalay
+aro Araona
+qwa Quechua, Corongo Ancash
+gga Gao
+zau Zangskari
+brt Bitare
+tyz Tày
+keu Akebu
+anm Anal
+lro Laro
+ssb Sama, Southern
+der Deori
+kad Adara
+esk Inupiatun, Northwest Alaska
+clo Chontal, Lowland Oaxaca
+bli Bolia
+tuz Turka
+bra Braj Bhasha
+nnm Namia
+sui Suki
+tgs Nume
+gbe Niksek
+xwe Gbe, Xwela
+kfp Korwa
+apt Apatani
+dzl Dzalakha
+mpq Matís
+hal Halang
+bio Nai
+jib Jibu
+kph Kplang
+hia Lamang
+yij Yindjibarndi
+chq Chinantec, Quiotepec
+xbi Kombio
+mpc Mangarrayi
+ebo Teke-Eboo
+tcs Torres Strait Creole
+kvi Kwang
+zyg Zhuang, Yang
+bww Bwa
+kpl Kpala
+hoy Holiya
+nhp Nahuatl, Isthmus-Pajapan
+abo Abon
+dai Day
+zom Zo
+lea Lega-Shabunda
+kej Kadar
+aup Makayam
+tcx Toda
+kmi Kami
+jio Jiamao
+bhd Bhadrawahi
+cav Cavineña
+bda Bayot
+ppq Pefiyahe
+bbk Babanki
+apu Apurinã
+ahr Ahirani
+wsi Wusi
+tdj Tajio
+myu Mundurukú
+kzq Kaike
+bfu Gahri
+sgh Shughni
+kfg Kudiya
+bcn Bali
+ygw Yagwoia
+ttv Titan
+iyo Mesaka
+pcn Abishi
+lkt Lakota
+aim Aimol
+tcf Me’phaa, Malinaltepec
+fod Foodo
+phk Phake
+scu Shumcho
+lch Luchazi
+nbm Ngbaka Ma’bo
+bei Bakati’
+jid Bu
+sce Dongxiang
+noi Noiri
+hmj Ge
+tyr Tai Daeng
+rop Kriol
+tsv Tsogo
+nbr Numana
+kvx Koli, Parkari
+ums Pendau
+dka Dakpakha
+alu ’Are’are
+pid Piaroa
+mab Mixtec, Yutanduchi
+gaq Gata’
+kgy Kyerung
+abs Malay, Ambonese
+alk Alak
+gdn Umanakaina
+ths Thakali
+khn Khandesi
+gaw Nobonob
+aac Ari
+tvd Tsuvadi
+bkr Bakumpai
+xkb Nago, Northern
+aot Atong
+lmn Lambadi
+kgr Abun
+moc Mocoví
+mbk Malol
+sss So
+dbv Dungu
+ngt Kriang
+tja Tajuasohn
+kif Kham, Eastern Parbate
+okv Orokaiva
+qvi Quichua, Imbabura Highland
+esu Yupik, Central
+bby Befang
+koi Komi-Permyak
+cvg Chug
+gdr Wipi
+kxp Koli, Wadiyari
+mme Mae
+pmj Pumi, Southern
+suy Suyá
+vas Vasavi
+suo Bouni
+nbc Naga, Chang
+bvr Burarra
+tts Thai, Northeastern
+diu Gciriku
+ndx Nduga
+bkl Berik
+lhp Lhokpu
+alf Elege
+wog Wogamusin
+bxa Bauro
+xwl Gbe, Western Xwla
+jae Yabem
+xbr Kambera
+bwd Bwaidoka
+nar Iguta
+dcc Deccan
+bjx Kalinga, Vanaw
+yes Nyankpa
+kul Kulere
+ssi Sansi
+hre Hre
+mtt Mota
+ysp Lolopo, Southern
+auc Waorani
+thy Tha
+dza Tunzuii
+tkb Buksa
+lkr Päri
+skn Subanon, Kolibugan
+tgd Ciwogai
+myp Pirahã
+eve Even
+bgg Bugun
+ril Riang Lang
+dbm Duguri
+bew Betawi
+aps Orop
+aon Weri
+dub Dubli
+hld Halang Doan
+jwi Jwira-Pepesa
+ayg Ginyanga
+wno Wano
+bfr Bazigar
+kpk Kpan
+bcg Baga Pokur
+avt Au
+nke Duke
+stk Aramba
+mkz Makasae
+hms Miao, Southern Qiandong
+duh Dungra Bhil
+scl Shina
+bfm Mmen
+ctl Chinantec, Tlacoatzintepec
+kra Kumal
+hmg Miao, Southwestern Guiyang
+zay Zayse
+faa Fasu
+lpn Naga, Long Phuri
+bqv Koro Wachi
+mpt Mian
+zak Zanaki
+pne Penan, Western
+apn Apinayé
+sbx Seberuang
+anp Angika
+bdv Bodo Parja
+juy Juray
+dso Desiya
+ndd Nde-Nsele-Nta
+ich Etkywan
+bkc Baka
+lez Lezgi
+lsh Lish
+mig Mixtec, San Miguel el Grande
+bdi Burun
+buu Budu
+ktn Karitiâna
+lbo Laven
+spn Sanapaná
+kgj Kham, Gamal
+kky Guugu Yimidhirr
+bjj Kanauji
+hve Huave, San Dionisio del Mar
+ghs Guhu-Samane
+vav Varli
+pih Pitcairn-Norfolk
+pcg Paniya
+ldj Lemoro
+brr Birao
+emn Eman
+lhl Lohar, Lahul
+pnc Pannei
+mnl Tiale
+ncq Katang, Northern
+xac Kachari
+xsn Sanga
+muz Mursi
+gwd Ale
+saf Safaliba
+dir Dirim
+dmg Kinabatangan, Upper
+isu Isu
+tpq Kinnauri, Chhoyul
+yuf Havasupai-Walapai-Yavapai
+oub Glio-Oubi
+ngn Ngwo
+fai Faiwol
+moi Mboi
+muo Mubako
+cih Chinali
+wew Wejewa
+luj Luna
+lkh Lakha
+wti Berta
+mse Musey
+bwo Borna
+nxr Ninggerum
+gru Kistane
+wiu Witu
+ndr Ndoola
+kmo Kwoma
+ksm Kumba
+ggb Gbii
+tqu Touo
+gia Kija
+aol Alor
+ute Ute-Southern Paiute
+xtj Mixtec, San Juan Teita
+khj Kuturmi
+bvh Bure
+kwc Likwala
+doz Dorze
+kga Koyaga
+cqd Miao, Chuanqiandian Cluster
+cjv Chuave
+hmb Songhay, Humburi Senni
+nac Narak
+iws Iwam, Sepik
+kxw Konai
+kmy Koma
+tww Tuwari
+arg Aragonese
+tig Tigré
+irx Kamberau
+ktv Katu, Eastern
+cdh Chambeali
+tis Itneg, Masadiit
+yeu Yerukula
+nzy Nzakambay
+drg Rungus
+wau Waurá
+mln Malango
+rmb Rembarrnga
+ldb Duya
+mjs Miship
+baw Bambili-Bambui
+dmo Kemedzung
+qxs Qiang, Southern
+kjq Keres, Western
+kwa Dâw
+azo Awing
+cjk Chokwe
+jeh Jeh
+drs Gedeo
+arh Arhuaco
+zdj Comorian, Ngazidja
+yaq Yaqui
+gyz Gyaazi
+fir Firan
+hbn Heiban
+ayb Gbe, Ayizo
+yde Yangum Dey
+gby Gbari
+byc Ubaghara
+bac Badui
+nhb Beng
+nms Letemboi
+pll Palaung, Shwe
+bwe Karen, Bwe
+ibb Ibibio
+agl Fembe
+nnp Naga, Wancho
+wmd Mamaindê
+kmt Kemtuik
+wja Waja
+bol Bole
+bhf Odiai
+xty Mixtec, Yoloxóchitl
+sgz Sursurunga
+apj Apache, Jicarilla
+drd Darmiya
+mqu Mandari
+brd Baram
+oym Wayampi
+uis Uisai
+eot Beti
+zpk Zapotec, Tlacolulita
+lbn Rmeet
+nqg Nago, Southern
+sme Saami, North
+zaz Zari
+sen Sénoufo, Nanerigé
+pca Popoloca, Santa Inés Ahuatempan
+biz Baloi
+brb Brao
+ppo Folopa
+amb Ambo
+krx Karon
+kwn Kwangali
+yiq Miqie
+gmb Gula’alaa
+res Reshe
+plc Palawano, Central
+bab Bainouk-Gunyuño
+kvb Kubu
+ymk Makwe
+nxk Naga, Kokak
+nut Nung
+dio Dibo
+tva Vaghua
+aez Aeka
+aoe Angal Enen
+bqh Baima
+otx Otomi, Texcatepec
+gdf Guduf-Gava
+mfl Putai
+adl Adi, Galo
+yay Agwagwune
+gas Garasia, Adiwasi
+aio Aiton
+tkx Tangko
+brf Bira
+usi Usoi
+vmz Mazatec, Mazatlán
+hru Hruso
+nja Nzanyi
+mfn Mbembe, Cross River
+ekr Yace
+nud Gala
+otr Otoro
+fie Fyer
+kwe Kwerba
+mgb Mararit
+yno Yong
+bef Benabena
+nux Mehek
+sto Stoney
+fqs Momu
+sbn Sindhi Bhil
+liq Libido
+jbm Bijim
+bfh Blafe
+isi Nkem-Nkum
+vig Viemo
+heg Helong
+kvl Kayaw
+thz Tamajeq, Tayart
+rin Nungu
+nco Sibe
+siw Motuna
+vmc Mixtec, Juxtlahuaca
+lev Pantar, Western
+mvn Minaveha
+tpl Me’phaa, Tlacoapa
+uiv Iyive
+pua Purepecha, Western Highland
+rnd Ruund
+cjm Cham, Eastern
+sym Samo, Maya
+bbt Mburku
+gvn Kuku-Yalanji
+kbx Ap Ma
+nsm Naga, Sumi
+bys Burak
+tlf Telefol
+mzq Mori Atas
+pck Chin, Paite
+hoo Holoholo
+wrm Warumungu
+tek Teke, Ibali
+zkr Zakhring
+ywl Lalu, Western
+mjw Karbi
+kmn Awtuw
+khs Kasua
+bnj Bangon
+mfd Mendankwe-Nkwen
+pqa Pa’a
+swo Shanenawa
+los Loniu
+nma Naga, Maram
+dgz Daga
+stj Samo, Matya
+ayu Ayu
+mxs Mixtec, Huitepec
+bpn Dzao Min
+tlx Khehek
+nbn Kuri
+ynq Yendang
+grh Tugbiri-Niragu
+juo Jiba
+amu Amuzgo, Guerrero
+myw Muyuw
+ybj Hasha
+hio Tshuwau
+kix Naga, Khiamniungan
+pma Paama
+bej Bedawiyet
+dni Dani, Lower Grand Valley
+naq Khoekhoe
+mrq Marquesan, North
+mrn Cheke Holo
+dgh Dghwede
+bau Mbat
+ite Itene
+crw Chrau
+ndb Kenswei Nsei
+nuk Nuu-chah-nulth
+dnd Daonda
+nlu Nchumbulu
+sge Segai
+can Chambri
+sre Bakati’, Sara
+kfr Kacchi
+hul Vula’a
+kid Koshin
+cyo Cuyonon
+ykm Kap
+ktm Kurti
+bsf Bauchi
+pio Piapoco
+kkc Odoodee
+thm Aheu
+xkn Kayan, Kayan River
+gfk Patpatar
+gel ut-Ma’in
+bsh Kati
+pmq Pame, Northern
+bfj Bafanji
+xwg Kwegu
+sng Sanga
+szp Suabo
+fvr Fur
+zwa Zay
+svs Savosavo
+chw Chuwabu
+nlv Nahuatl, Orizaba
+bsp Baga Sitemu
+bdl Bajau, Indonesian
+khy Kele
+ito Itonama
+naj Nalu
+bdd Bunama
+emb Embaloh
+zps Zapotec, Coatlán
+kee Keres, Eastern
+ukw Ukwuani-Aboh-Ndoni
+ldm Landoma
+duw Dusun Witu
+mxp Mixe, Tlahuitoltepec
+zln Zhuang, Lianshan
+zns Mangas
+blf Buol
+ksn Kasiguranin
+prm Kibiri
+lmd Lumun
+lop Lopa
+yev Yeri
+kwk Kwakwala
+tcp Chin, Tawr
+int Intha
+clj Chin, Laitu
+jit Jita
+mgc Morokodo
+ags Esimbi
+tvu Tunen
+ghk Karen, Geko
+hue Huave, San Francisco del Mar
+kkn Kon Keu
+pbm Mazatec, Puebla and Northeastern
+snl Sangil
+jkp Karen, Paku
+yrk Nenets
+ciw Chippewa
+mlf Mal
+pym Pyam
+vrs Varisi
+nnd Ambae, West
+akg Anakalangu
+udi Udi
+kys Kayan, Baram
+lky Lokoya
+bui Bongili
+zkd Kadu
+ihp Iha
+cdr Kamuku
+anj Anor
+ndm Ndam
+lga Lungga
+hmw Miao, Western Mashan
+zkn Kanan
+bpz Bilba
+taw Tay
+mez Menominee
+wuv Wuvulu-Aua
+mkk Byep
+aki Aiome
+gue Gurindji
+bse Wushi
+dsq Tadaksahak
+spt Spiti Bhoti
+hoj Haroti
+aom Ömie
+mdt Mbere
+nbb Ndoe
+ape Bukiyip
+eky Kayah, Eastern
+itd Tidung, Southern
+mcc Bitur
+kzi Kelabit
+bhq Tukang Besi South
+dia Dia
+asb Assiniboine
+wyy Fijian, Western
+nna Nyangumarta
+twx Tewe
+mlq Maninkakan, Western
+uta Itang
+hmz Sinicized Miao
+aof Bragat
+rue Rusyn
+pbs Pame, Central
+kio Kiowa
+tdn Tondano
+snm Ma’di, Southern
+cod Kukama-Kukamiria
+cde Chenchu
+ppl Nahuat
+tdg Tamang, Western
+jmb Zumbun
+eit Eitiep
+wni Comorian, Ndzwani
+tlp Totonac, Filomena Mata-Coahuitlán
+ilk Bogkalot
+nri Naga, Chokri
+kyv Kewat
+scs Slavey, North
+kji Zabana
+tku Totonac, Upper Necaxa
+byp Bumaji
+xkt Kantosi
+kcc Lubila
+yuq Yuqui
+cho Choctaw
+hot Malei
+kku Tumi
+bmi Bagirmi
+wlc Comorian, Mwali
+auu Auye
+wle Wolane
+mmm Maii
+cdo Chinese, Min Dong
+nez Nez Perce
+ukp Bukpe
+kwo Kwomtari
+zpx Zapotec, San Baltazar Loxicha
+mlv Mwotlap
+ppm Papuma
+bqr Burusu
+wut Wutung
+tji Tujia, Northern
+bbq Bamali
+ttk Totoro
+ets Etsako
+yin Riang Lai
+gim Gimi
+kow Kugama
+ksa Shuwa-Zamani
+git Gitxsan
+erk Efate, South
+vmx Mixtec, Tamazola
+duv Duvle
+bgf Bangandu
+wms Wambon
+isd Isnag
+pmx Naga, Poumai
+doy Dompo
+nak Nakanai
+bze Bozo, Jenaama
+gis Giziga, North
+miu Mixtec, Cacaloxtepec
+bzu Burmeso
+ckx Caka
+duu Drung
+jmr Kamara
+lur Laura
+wlw Walak
+rar Cook Islands Maori
+osi Osing
+mmd Maonan
+kmm Kom
+kvr Kerinci
+ncr Nchane
+for Fore
+bgn Balochi, Western
+gnm Ginuman
+alx Amol
+xks Kumbewaha
+lkn Lakon
+mbl Maxakalí
+bri Mokpwe
+mov Mohave
+pot Potawatomi
+pnu Bunu, Jiongnai
+djm Dogon, Jamsay
+ula Fungwa
+nnj Nyangatom
+ybl Yukuben
+aab Arum
+has Haisla
+alh Alawa
+mea Menka
+pum Puma
+spo Spokane
+cyb Cayubaba
+nbh Ngamo
+ont Ontenu
+ahp Aizi, Aproumu
+bpy Bishnupuriya
+utr Etulo
+auk Heiyoho
+bdb Basap
+klo Kapya
+nrf Guernésiais
+tmn Taman
+mvo Marovo
+kla Klamath-Modoc
+jnl Rawat
+jad Jahanka
+hrm Miao, Horned
+hoa Hoava
+mus Muskogee
+dna Dani, Upper Grand Valley
+btg Bété, Gagnoa
+ngs Gvoko
+lmu Lamenu
+add Lidzonka
+pha Pa-Hng
+kvq Karen, Geba
+pch Pardhan
+bgs Tagabawa
+nir Nimboran
+bcy Bacama
+var Huarijío
+sjo Xibe
+jle Ngile
+cuv Cuvok
+smf Auwe
+cnq Chung
+bhh Bukharic
+dox Mositacha
+ior Inor
+oma Omaha-Ponca
+abz Abui
+kza Karaboro, Western
+rbb Palaung, Rumai
+bfq Badaga
+kht Khamti
+sps Saposa
+syk Sukur
+slp Lamaholot
+jax Malay, Jambi
+byo Biyo
+qvj Quichua, Loja Highland
+bnx Bangubangu
+ngw Ngwaba
+krf Koro
+loa Loloda
+cox Nanti
+wwo Dorig
+akc Mpur
+kal Greenlandic
+siu Sinagen
+aqm Atohwaim
+rmt Domari
+nhn Nahuatl, Central
+jum Jumjum
+nix Hema
+ncg Nisga’a
+ccl Cutchi-Swahili
+kvu Yinbaw
+tnb Tunebo, Western
+cpx Chinese, Pu-Xian
+kgo Krongo
+nxd Ngando
+coj Cochimi
+grx Muno
+bfs Bai, Southern
+cov Cao Miao
+cbj Ede Cabe
+loh Narim
+iry Iraya
+cky Cakfem-Mushere
+bsn Barasana-Eduria
+xkv Kgalagadi
+itz Itza’
+tgc Tigak
+boh Boma
+mck Mbunda
+ccg Samba Daka
+piy Piya-Kwonci
+how Honi
+pwm Molbog
+tds Doutai
+ldg Lenyima
+csa Chinantec, Chiltepec
+cbk Chavacano
+ibl Ibaloi
+kql Kyenele
+smq Samo
+uya Doko-Uyanga
+tkd Tukudede
+cry Kyoli
+clk Idu-Mishmi
+cut Cuicatec, Teutila
+apm Apache, Mescalero-Chiricahua
+bya Batak
+nyi Ama
+nih Nyiha, Tanzania
+hbb Nya Huba
+huc ǂ’Amkhoe
+cdi Chodri
+rhp Yahang
+bcj Bardi
+pei Chichimeco-Jonaz
+gdl Dirasha
+emg Mewahang, Eastern
+mmz Mabaale
+afo Ajiri
+bhs Buwal
+lht Lo-Toga
+ktp Kaduo
+xns Kanashi
+sjb Sajau Basap
+pow Popoloca, San Felipe Otlaltepec
+rad Rade
+gut Maléku Jaíka
+vam Dumo
+kis Kis
+bet Bété, Guiberoua
+lva Makuva
+zoc Zoque, Copainalá
+goa Guro
+bkg Buraka
+yae Pumé
+won Wongo
+gpa Gupa-Abawa
+sde Vori
+mls Masalit
+jiu Jinuo, Youle
+bmd Baga Manduri
+czt Chin, Zotung
+tvn Tavoyan
+zng Mang
+ijj Ede Ije
+dms Dampelas
+mlw Moloko
+wow Wawonii
+png Pangu
+ikw Ikwere
+dtb Kadazan, Labuk-Kinabatangan
+bey Beli
+ntu Natügu
+sua Sulka
+kcx Kachama-Ganjule
+ekl Kol
+mhp Malay, Balinese
+slz Ma’ya
+skt Sakata
+dez Dengese
+ogc Ogbah
+byz Waran
+yui Wajiara
+kdx Kam
+erh Eruwa
+atp Atta, Pudtol
+qws Quechua, Sihuas Ancash
+ale Aleut
+lcm Tungag
+pbp Badyara
+anc Ngas
+khl Lusi
+mkc Siliput
+knm Kanamarí
+yah Yazgulyam
+hml Miao, Luopohe
+mfb Bangka
+mxl Gbe, Maxi
+lgt Pahi
+das Daho-Doo
+njo Naga, Ao
+iar Purari
+nou Ewage-Notu
+moh Mohawk
+tvl Tuvaluan
+yuy Yugur, East
+kvt Lahta
+sku Sakao
+hra Hrangkhol
+nka Nkoya
+crx Carrier
+tif Tifal
+pia Pima Bajo
+ppi Paipai
+nbp Nnam
+btm Batak Mandailing
+jya Jiarong
+mxd Modang
+psn Panasuan
+puc Punan Merap
+tty Sikaritai
+mzb Tumzabt
+zmb Zimba
+kdu Kadaru
+nnz Nda’nda’
+nmb V’ënen Taut
+tcd Tafi
+weh Weh
+jni Janji
+txn Tarangan, West
+pem Phende
+xod Kokoda
+byj Bina
+bpw Bo
+bbf Baibai
+sol Solos
+mmc Mazahua, Michoacán
+pta Pai Tavytera
+khc Tukang Besi North
+nau Nauruan
+llu Lau
+pnz Pana
+kuy Kuuku-Ya’u
+wbq Waddar
+wud Wudu
+mbi Manobo, Ilianen
+ikt Inuinnaqtun
+bhp Bima
+mdj Mangbetu
+swj Sira
+xom Komo
+rir Ribun
+sbr Murut, Sembakung
+tfn Tanaina
+pwa Pawaia
+msw Mansoanka
+zpn Zapotec, Santa Inés Yatzechi
+rkm Marka
+aun One, Molmo
+mxa Mixtec, Northwest Oaxaca
+abr Abron
+bxs Busam
+bly Notre
+gro Groma
+mvz Mesqan
+yum Quechan
+nxg Ngad’a
+akw Akwa
+kmp Gimme
+kfh Kurichiya
+ged Gade
+yuj Karkar-Yuri
+hwo Hwana
+pkt Maleng
+agc Agatu
+mgi Migili
+akt Akolet
+bkw Bekwel
+dun Dusun Deyah
+mrh Chin, Mara
+dgd Dagaari Dioula
+kci Kamantan
+yak Yakama
+kch Vono
+bxq Beele
+chx Chantyal
+gra Garasia, Rajput
+kih Kilmeri
+ono Onondaga
+adn Adang
+aug Aguna
+bqt Bamukumbit
+mum Maiwala
+atu Reel
+hop Hopi
+bhy Bhele
+zms Mbesa
+prx Purig
+bjp Fanamaket
+odu Odual
+azd Nahuatl, Eastern Durango
+bje Biao-Jiao Mien
+mct Mengisa
+njm Naga, Angami
+liu Logorik
+pwn Paiwan
+mav Sateré-Mawé
+gnu Gnau
+jub Wannu
+sez Chin, Senthang
+mgg Mpumpong
+ost Osatu
+vkl Kulisusu
+kbj Kari
+bag Tuki
+bjt Balanta-Ganja
+mkf Miya
+ngi Ngizim
+mds Maria
+gvf Golin
+thd Kuuk Thayorre
+rau Raute
+sse Sama, Balangingih
+nhz Nahuatl, Santa María la Alta
+cvn Chinantec, Valle Nacional
+nba Nyemba
+hnd Hindko, Southern
+nbi Naga, Mao
+bil Bille
+xmh Kugu-Muminh
+bip Bila
+zhi Zhire
+aal Afade
+mfg Mogofin
+wan Wan
+kkf Monpa, Kalaktang
+nyq Nayini
+ors Orang Seletar
+bbp Banda, West Central
+yle Yélî Dnye
+taz Tocho
+dri C’Lela
+nbv Ngamambo
+mqg Malay, Kota Bangun Kutai
+mdu Mboko
+aty Aneityum
+mbq Maisin
+hav Havu
+ner Yahadian
+glw Glavda
+nyb Nyangbo
+clt Chin, Lautu
+jiy Jinuo, Buyuan
+qxa Quechua, Chiquián
+win Ho-Chunk
+chr Cherokee
+vkn Koro Nulu
+quv Sakapulteko
+wrs Waris
+nit Kolami, Southeastern
+ver Verre
+nmk Namakura
+czh Chinese, Huizhou
+wrp Waropen
+mmx Madak
+yis Yis
+kce Kaivi
+ddg Fataluku
+sle Sholaga
+ega Ega
+jnd Jandavra
+kxx Likuba
+kna Dera
+yer Tarok
+amm Sawiyanu
+onn Onobasulu
+tce Tutchone, Southern
+buh Bunu, Younuo
+sst Sinasina
+wsk Waskia
+dln Darlong
+teq Temein
+org Oring
+cfg Karimjo
+nce Yale
+lgq Logba
+yif Ache
+kfo Koro
+jog Jogi
+nkx Nkoroo
+asr Asuri
+ktc Kholok
+gbz Dari, Zoroastrian
+kvy Yintale
+kvv Kola
+oia Oirata
+rdb Rudbari
+ymb Yambes
+sad Sandawe
+ntk Ikoma-Nata-Isenye
+dru Rukai
+bjh Bahinemo
+ywa Kalou
+nmc Ngam
+nat Cahungwarya
+ato Atong
+liw Col
+qux Quechua, Yauyos
+shw Shwai
+cfd Cara
+pip Pero
+zts Zapotec, Tilquiapan
+mcs Mambai
+sgi Nizaa
+mhl Mauwake
+ndu Dugun
+bqa Tchumbuli
+bqo Balo
+buz Bukwen
+fak Fang
+tii Tiene
+gvp Gavião, Pará
+kmh Kalam
+xkc Kho’ini
+max Malay, North Moluccan
+phl Palula
+gbg Gbanziri
+zag Zaghawa
+trf Trinidadian English Creole
+weo Wemale
+geg Gengle
+kxb Krobu
+pru Puragi
+kie Kibet
+mpn Mindiri
+mhz Mor
+gbh Gbe, Defi
+gbr Gbagyi
+tmy Tami
+rey Reyesano
+kpj Karajá
+nap Napoletano-Calabrese
+lgu Longgu
+bye Pouye
+tml Citak, Tamnim
+kpw Kobon
+kfa Kodava
+iyx Yaka
+twy Tawoyan
+sed Sedang
+bdm Buduma
+plg Pilagá
+buo Terei
+aww Awun
+yyu Yau
+cld Chaldean Neo-Aramaic
+xmg Mengaka
+pku Paku
+xkg Kagoro
+caq Nicobarese, Car
+kmq Gwama
+lel Lele
+gqa Ga’anda
+tfi Gbe, Tofin
+mml Man Met
+nxa Nauete
+tdk Rom
+kbv Dla
+bgv Warkay-Bipim
+bbw Supapya
+kvm Kendem
+aku Akum
+ert Eritai
+jdg Jadgali
+gow Gorowa
+doo Dongo
+jeb Jebero
+stf Seta
+nid Ngandi
+mqx Mamuju
+mta Manobo, Cotabato
+she Sheko
+mfm Marghi South
+jei Yei
+deg Degema
+gcf Guadeloupean French Creole
+bxb Belanda Bor
+mut Muria, Western
+diw Dinka, Northwestern
+nqy Naga, Akyaung Ari
+sop Songe
+kny Kanyok
+lse Lusengo
+ahg Qimant
+opa Okpamheri
+hah Hahon
+daq Maria, Dandami
+hac Gurani
+klg Tagakaulo
+kqi Koita
+slx Salampasu
+ots Otomí, Estado de México
+tru Turoyo
+sxw Gbe, Saxwe
+dij Dai
+aog Angoram
+kcr Katla
+agf Arguni
+alq Algonquin
+raf Mewahang, Western
+mij Mungbam
+gdu Gudu
+wgi Wahgi
+bbu Kulung
+ndo Ndonga
+mma Mama
+tal Tal
+odk Oadki
+etr Edolo
+umu Munsee
+kjs Kewapi, East
+bvm Bamunka
+jqr Jaqaru
+kfm Khunsari
+tbp Diebroud
+ems Yupik, Pacific Gulf
+kcq Kamo
+ruy Mala
+nng Naga, Maring
+jns Jaunsari
+sbk Safwa
+wji Warji
+sbz Sara Kaba
+bhl Bimin
+auy Awiyaana
+txt Citak
+nof Nomane
+cll Chala
+pak Parakanã
+tli Tlingit
+kqo Krahn, Eastern
+kbz Duhwa
+mbx Mari
+xrw Karawa
+crj Cree, Southern East
+jaq Yaqay
+pbn Kpasham
+dbi Doka
+kod Kodi
+bjk Barok
+syb Subanen, Central
+nyh Nyikina
+kfd Koraga, Korra
+mtk Mbo’
+mbd Manobo, Dibabawon
+jgk Gwak
+mmp Siawi
+uba Ubang
+kxh Karo
+tov Taromi, Upper
+buk Bugawac
+abn Abua
+kbh Camsá
+slc Sáliba
+knt Katukína, Panoan
+rwa Rawo
+kyk Kamayo
+kli Kalumpang
+klq Rumu
+iqu Iquitu
+jku Labir
+bga Gwamhi-Wuri
+amo Amo
+gou Gavar
+kdz Kwaja
+nzm Naga, Zeme
+mgk Mawes
+sjr Siar-Lak
+aqg Arigidi
+ghl Ghulfan
+oso Ososo
+rei Reli
+tiw Tiwi
+kdq Koch
+zbu Bu
+wem Gbe, Weme
+gig Goaria
+tsw Tsishingini
+gmz Mgbolizhia
+mfo Mbe
+anw Anaang
+mtu Mixtec, Tututepec
+ahb Axamb
+bub Bua
+jru Japreria
+ryu Okinawan, Central
+nuo Nguôn
+kdm Gyong
+due Agta, Umiray Dumaget
+boo Bozo, Tiemacèwè
+vmm Mixtec, Mitlatongo
+ydg Yadgha
+adz Adzera
+yaf Yaka
+mep Miriwoong
+kip Kham, Sheshi
+bvw Boga
+mqh Mixtec, Tlazoyaltepec
+bmj Bote
+dih Kumiai
+cib Gbe, Ci
+ggg Gurgula
+ldq Lufu
+scv Sheni
+siy Sivandi
+ktf Kwami
+gew Gera
+lan Laru
+kks Giiwo
+fun Iatê
+dtm Dogon, Tomo Kan
+thp Thompson
+gye Gyem
+zaf Zapotec, Ayoquesco
+kcs Koenoem
+yap Yapese
+bnv Beneraf
+src Sardinian, Logudorese
+brq Breri
+frc French, Cajun
+elk Elkei
+aad Amal
+kqj Koromira
+ael Ambele
+mku Maninka, Konyanka
+otm Otomi, Eastern Highland
+ldp Tso
+dbd Dadiya
+ttm Tutchone, Northern
+nen Nengone
+bit Berinomo
+wca Yanomámi
+jig Jingulu
+wss Wasa
+huh Huilliche
+xti Mixtec, Sinicahua
+nhv Nahuatl, Temascaltepec
+smy Semnani
+tak Tala
+hch Huichol
+kqa Mum
+spm Akukem
+kfz Koromfé
+ank Goemai
+ruz Ruma
+koh Koyo
+pdo Padoe
+kvd Kui
+fut Futuna-Aniwa
+wom Wom
+sor Soumraye
+gdx Godwari
+ttb Gaa
+iti Itneg, Inlaud
+tsp Toussian, Northern
+jkr Koro
+sct Katang, Southern
+laa Subanen, Southern
+auq Anus
+agy Alta, Southern
+tuq Tedaga
+acv Achumawi
+mbv Mbulungish
+orh Oroqen
+def Dezfuli
+gop Yeretuar
+nyg Nyindu
+liz Libinza
+tay Atayal
+dil Dilling
+mtf Murik
+jup Hupdë
+uuu U
+ncf Notsi
+hum Hungana
+vum Vumbu
+mfj Mefele
+afi Chini
+meh Mixtec, Southwestern Tlaxiaco
+tma Tama
+mkg Mak
+aik Akye
+ung Ngarinyin
+itt Itneg, Maeng
+akl Aklanon
+sti Stieng, Bulo
+gid Gidar
+ckl Kibaku
+spu Sapuan
+enn Engenni
+ebr Tchaman
+mcw Mawa
+ybe Yugur, West
+kni Kanufi
+kjr Kurudu
+bwm Biwat
+vra Vera’a
+duq Dusun Malang
+bpu Bongu
+mrz Marind
+sdh Kurdish, Southern
+cdn Chaudangsi
+vmp Mazatec, Soyaltepec
+zsm Malay, Standard
+szg Sengele
+yun Bena
+kcd Kanum, Ngkâlmpw
+ala Alago
+ywn Yawanawa
+nfl Äiwoo
+pbl Mak
+pyu Puyuma
+zrg Mirgan
+aif Agi
+kmj Kumarbhag Paharia
+njj Njen
+ahs Ashe
+kwu Kwakum
+mxh Mvuba
+chp Dene
+iko Olulumo-Ikom
+krh Kurama
+bux Boghom
+udl Wuzlam
+one Oneida
+akq Ak
+fla Kalispel-Pend d’Oreille
+zpr Zapotec, Santiago Xanica
+tvt Naga, Tutsa
+awe Awetí
+bqx Baangi
+yns Iyansi
+dya Dyan
+hkk Hunjara-Kaina Ke
+clc Chilcotin
+kpa Kutto
+ldk Leelau
+dak Dakota
+vls West Flemish
+xnz Mattokki
+ccj Kasanga
+kzc Kulango, Bondoukou
+dkx Mazagway-Hidi
+leq Lembena
+saz Saurashtra
+mqz Pano
+akr Araki
+fap Paloor
+mef Megam
+rat Razajerdi
+kmk Kalinga, Limos
+ike Inuktitut, Eastern Canadian
+see Seneca
+nlo Ngul
+klk Kono
+rcf Réunion French Creole
+bof Bolon
+rwk Rwa
+smt Simte
+jma Dima
+mmn Minamanwa
+mhk Mungaka
+whg Yuwei
+zro Záparo
+sob Sobei
+mtp Weenhayek
+zuy Zumaya
+ocu Matlatzinca, Atzingo
+xtt Mixtec, Tacahua
+mek Mekeo
+ctt Chetti, Wayanad
+bni Bangi
+ogb Ogbia
+orx Oro
+kot Lagwan
+itr Yawuno Teneyo
+kic Kickapoo
+skd Miwok, Southern Sierra
+nhg Nahuatl, Tetelcingo
+bvi Belanda Viri
+tny Tongwe
+rui Rufiji
+dor Dori’o
+lmk Lamkang
+ncb Nicobarese, Central
+msg Moraid
+snq Sangu
+eme Tekó
+amc Amahuaca
+msn Vurës
+hdn Haida, Northern
+com Comanche
+sgd Surigaonon
+cdf Chiru
+ttj Tooro
+skv Skou
+twp Ere
+gek Ywom
+cob Chicomuceltec
+fll Fali, North
+mne Naba
+coc Cocopa
+mph Maung
+gaf Gende
+agh Ngelima
+epi Epie
+aaw Solong
+sok Sokoro
+piu Pintupi-Luritja
+dyg Agta, Villa Viciosa
+mla Malo
+dof Domu
+klx Koluwawa
+gab Gabri
+scn Sicilian
+mat Matlatzinca, San Francisco
+bja Budza
+kcj Kobiana
+kwb Kwa
+tsu Tsou
+kev Kanikkaran
+ksj Uare
+zrs Mairasi
+bcv Shoo-Minda-Nye
+sug Suganga
+pcl Pardhi
+yim Naga, Yimchungru
+kqk Gbe, Kotafon
+bzx Bozo, Kelengaxo
+esh Eshtehardi
+fay Fars, Southwestern
+dee Dewoin
+eze Uzekwe
+bwt Bafaw-Balong
+nph Naga, Phom
+pmm Pol
+pdc German, Pennsylvania
+srz Shahmirzadi
+tug Tunia
+hux Witoto, Nipode
+soo Songo
+bcz Bainouk-Gunyaamolo
+bva Barein
+sky Sikaiana
+blc Bella Coola
+skq Sininkere
+yix Axi
+arx Aruá
+msl Molof
+aqt Angaité
+gcr Guianese French Creole
+mtb Anyin Morofo
+lrl Lari
+tiy Teduray
+iwm Iwam
+bhg Binandere
+pbv Pnar
+gmm Gbaya-Mbodomo
+apy Apalaí
+iow Iowa-Oto
+cku Koasati
+sry Sera
+zcd Zapotec, Las Delicias
+toj Tojolabal
+idi Idi
+kqw Kandas
+irr Ir
+bif Biafada
+akf Akpa
+arw Arawak
+lor Téén
+was Washo
+nrg Narango
+knz Kalamsé
+anf Animere
+goz Gozarkhani
+vmh Maraghei
+arp Arapaho
+glr Glaro-Twabo
+big Biangai
+tou Tho
+lie Likila
+hol Holu
+dbn Duriankere
+asu Asurini, Tocantins
+xvi Kamviri
+aaf Aranadan
+mii Mixtec, Chigmecatitlán
+xkj Kajali
+bez Bena
+trv Seediq
+bqs Bosmun
+yax Yauma
+ykg Yukaghir, Northern
+hgm Hai|ǁom
+sgr Sangisari
+vaf Vafsi
+anl Chin, Anu-Khongso
+mdh Maguindanaon
+bbv Karnai
+wbb Wabo
+shc Sonde
+nsa Naga, Sangtam
+rtm Rotuman
+kvg Kuni-Boazi
+cgg Chiga
+mdn Mbati
+job Joba
+bxl Jalkunan
+jrt Jakattoe
+ilp Iranun
+njh Naga, Lotha
+sek Sekani
+avi Avikam
+nmh Naga, Monsang
+cos Corsican
+ctz Chatino, Zacatepec
+wbj Alagwa
+sbg Seget
+tyy Tiyaa
+bea Beaver
+chd Chontal, Highland Oaxaca
+ado Abu
+mnv Rennell-Bellona
+dbb Deno
+mti Maiwa
+ekp Ekpeye
+plr Sénoufo, Palaka
+nev Nyaheun
+cra Chara
+tla Tepehuan, Southwestern
+xmf Mingrelian
+nyw Nyaw
+dis Dimasa
+zpy Zapotec, Mazaltepec
+dgx Doghoro
+ifm Teke-Wuumu
+ngz Ngungwel
+yra Yerakai
+sau Saleman
+psw Port Sandwich
+kbm Iwal
+mye Myene
+tiq Tiéfo
+kkh Khün
+kjt Karen, Phrae Pwo
+gox Gobu
+kzm Kais
+pac Pacoh
+gua Shiki
+too Totonac, Xicotepec de Juárez
+nre Naga, Southern Rengma
+pqm Malecite-Passamaquoddy
+gul Sea Island English Creole
+cte Chinantec, Tepinapa
+buf Bushoong
+bws Bomboma
+tlq Tai Loi
+asi Buruwai
+bpv Marind, Bian
+atk Ati
+gar Galeya
+plv Palawano, Southwest
+sev Sénoufo, Nyarafolo
+vem Vemgo-Mabas
+hla Halia
+mna Mbula
+pcb Pear
+lih Lihir
+ksv Kusu
+iby Ibani
+yrb Yareba
+nge Ngemba
+ney Neyo
+keb Kélé
+nuq Nukumanu
+okh Koresh-e Rostam
+ity Itneg, Moyadan
+van Walman
+ijs Ijo, Southeast
+shs Shuswap
+mkb Mal Paharia
+kit Agob
+nyj Nyanga
+tti Tobati
+agb Legbo
+twr Tarahumara, Southwestern
+cae Laalaa
+biu Biate
+grs Gresi
+brp Barapasi
+tdv Atoro
+crv Chaura
+njs Nisa
+oke Okpe
+tdl Kusur-Myet
+mlx Na’ahai
+zte Zapotec, Elotepec
+ivb Ibatan
+chy Cheyenne
+mbf Malay, Baba
+nal Nalik
+lwl Lawa, Eastern
+buw Bubi
+qus Quichua, Santiago del Estero
+lik Lika
+lna Langbashe
+dem Dem
+ldo Loo
+pbg Paraujano
+mic Mi’kmaq
+wdj Wadjiginy
+tol Tolowa
+sns Nahavaq
+luz Luri, Southern
+tgy Togoyo
+sha Shall-Zwall
+mtl Tehl
+scw Sya
+hna Mina
+moe Innu
+mae Bo-Rukul
+avd Alviri-Vidari
+bsy Bisaya, Sabah
+kfe Kota
+dsn Dusner
+kst Winyé
+bid Bidiyo
+erg Sie
+tls Tambotalo
+nkw Nkutu
+zia Zia
+bdw Baham
+une Uneme
+ykk Yakaikeke
+plu Palikúr
+pfe Pere
+blq Paluai
+sao Sause
+tsa Tsaangi
+uni Uni
+irn Irántxe
+pos Popoluca, Sayula
+mot Barí
+lki Laki
+gbn Mo’da
+chk Chuukese
+kmz Khorasani Turkish
+orz Ormu
+bfe Betaf
+nlj Nyali
+bnn Bunun
+aba Abé
+abu Abure
+iai Iaai
+knn Konkani
+biy Birhor
+yog Yogad
+gnb Gangte
+bou Bondei
+zmq Mituku
+tto Ta’oih, Lower
+abm Abanyom
+dhv Drehu
+brl Birwa
+shh Shoshoni
+zbc Berawan, Central
+oyd Oyda
+pek Penchal
+tbj Tiang
+ema Emai-Iuleha-Ora
+bgi Bagobo-Klata
+tkq Tee
+nmm Nyeshangte
+kkk Kokota
+djn Djauan
+tow Jemez
+nwm Nyamusa-Molo
+tef Teressa
+daw Davawenyo
+kpx Koiali, Mountain
+mtq Muong
+mwe Mwera
+stv Silt’e
+lum Luimbi
+phq Phana’
+tdc Embera Tadó
+pcj Parenga
+vnk Lovono
+kdd Yankunytjatjara
+aul Aulua
+mnp Chinese, Min Bei
+tdo Teme
+mwg Aiklep
+dma Duma
+coz Chocholtec
+owi Owiniga
+rji Raji
+aey Amele
+dge Degenang
+nil Nila
+ler Lenkau
+agt Agta, Central Cagayan
+kof Kubi
+okx Okpe
+ogg Ogbogolo
+xes Koromu
+hur Halkomelem
+bgx Balkan Gagauz Turkish
+anx Andra-Hus
+rwo Rawa
+caz Canichana
+kuh Kushi
+bks Sorsoganon, Northern
+ztx Zapotec, Zaachila
+axk Yaka
+umm Umon
+mmy Migaama
+aee Pashai, Northeast
+lil Lillooet
+lvk Lavukaleve
+ibd Iwaidja
+azt Atta, Faire
+usa Usarufa
+saa Saba
+bar Bavarian
+mzn Mazandarani
+unx Munda
+puo Puoc
+lek Leipon
+pkg Pak-Tong
+niu Niue
+oni Onin
+jaf Jara
+dwa Diri
+lmg Lamogai
+tau Tanana, Upper
+zoh Zoque, Chimalapa
+cbg Chimila
+gla Scottish Gaelic
+yur Yurok
+peb Pomo, Eastern
+kbb Kaxuiâna
+ivv Ivatan
+oka Okanagan
+ral Ralte
+nun Anong
+soz Temi
+ndh Ndali
+kdy Keijar
+bjo Banda, Mid-Southern
+env Enwan
+nds Saxon, Low
+kyy Asa’a
+moy Shekkacho
+mnm Mapena
+sbh Sori-Harengan
+bek Bebeli
+pdn Fedan
+mxm Meramera
+moj Monzombo
+tul Tula
+oks Oko-Eni-Osayen
+bjc Bariji
+hvv Huave, Santa María del Mar
+dme Dugwor
+plk Shina, Kohistani
+lal Lalia
+sir Siri
+yhd Arabic, Judeo-Iraqi
+zmp Mpuono
+ofu Efutop
+iki Iko
+sjg Assangori
+gae Guarequena
+hei Heiltsuk
+dmr Damar, East
+lti Leti
+ipo Ipiko
+dva Duau
+yaw Yawalapití
+dgg Doga
+mdw Mbosi
+mzv Mandja
+tkp Tikopia
+snv Sa’ban
+bte Gamo-Ningi
+nqt Nteng
+etx Iten
+gwa Mbato
+aji Ajië
+gni Gooniyandi
+blm Beli
+hid Hidatsa
+tof Gizrra
+kos Kosraean
+cja Cham, Western
+yki Yoke
+haa Han
+gad Gaddang
+mbs Manobo, Sarangani
+kkd Kinuku
+kol Kol
+noz Nayi
+kms Kamasau
+kfu Katkari
+mps Dadibi
+sbc Kele
+pon Pohnpeian
+sos Seeku
+kwt Kwesten
+diz Ding
+buj Basa-Gurmana
+onj Onjob
+cbo Izora
+pic Pinji
+zpw Zapotec, Zaniza
+sro Sardinian, Campidanese
+psi Pashai, Southeast
+kyh Karok
+msk Mansaka
+kxn Melanau, Kanowit-Tanjong
+end Ende
+mgf Maklew
+wgb Wagawaga
+mhc Mocho
+niw Nimo
+gyd Kayardild
+crc Lonwolwol
+bwf Boselewa
+huf Humene
+lad Ladino
+dei Demisa
+kgb Kawe
+prc Parachi
+ttw Long Wat
+ilu Ili’uun
+mnu Mer
+mbo Mbo
+glo Galambu
+sys Sinyar
+sgy Sanglechi
+poo Pomo, Central
+tsi Tsimshian
+svb Ulau-Suain
+wsa Warembori
+kkz Kaska
+gsw German, Swiss
+skb Saek
+ano Andoque
+zun Zuni
+tnm Tabla
+sbb Simbo
+wkd Mo
+sby Soli
+xok Xokleng
+chj Chinantec, Ojitlán
+jge Judeo-Georgian
+ugo Ugong
+lmi Lombi
+nkh Naga, Khezha
+huz Hunzib
+tft Ternate
+mrp Morouas
+mrf Elseng
+yot Yotti
+gbv Gbanu
+ayt Ayta, Magbukun
+hgw Haigwai
+swr Saweru
+lcc Legenyem
+zpe Zapotec, Petapa
+zpd Zapotec, Southeastern Ixtlán
+kep Kaikadi
+vmj Mixtec, Ixtayutla
+clu Caluyanun
+cma Maa
+qun Quinault
+kcf Ukaan
+fry Frisian
+har Harari
+bta Bata
+wro Worrorra
+mwp Kala Lagaw Ya
+sny Saniyo-Hiyewe
+nzk Nzakara
+knw Kung-Ekoka
+wbk Waigali
+smn Saami, Inari
+shq Sala
+zat Zapotec, Tabaa
+ngj Ngie
+psh Pashai, Southwest
+amt Amto
+xgu Unggumi
+qui Quileute
+gww Kwini
+agi Agariya
+caf Carrier, Southern
+pay Pech
+cbd Carijona
+mwa Mwatebu
+gcn Gaina
+suq Suri, Tirmaga-Chai
+djc Daju, Dar Daju
+aaa Ghotuo
+etn Eton

data/tts/all_langs.tsv

@@ -0,0 +1,1143 @@
+abi Abidji
+ace Aceh
+aca Achagua
+acn Achang
+acr Achi
+ach Acholi
+acu Achuar-Shiwiar
+guq Aché
+ade Adele
+adj Adioukrou
+agd Agarabi
+agx Aghul
+agn Agutaynen
+aha Ahanta
+aka Akan
+knj Akateko
+ake Akawaio
+aeu Akeu
+ahk Akha
+bss Akoose
+alj Alangan
+sqi Albanian
+alt Altai, Southern
+alp Alune
+alz Alur
+kab Amazigh
+amk Ambai
+mmg Ambrym, North
+amh Amharic
+ami Amis
+azg Amuzgo, San Pedro Amuzgos
+agg Angor
+boj Anjam
+cko Anufo
+any Anyin
+arl Arabela
+ara Arabic
+atq Aralle-Tabulahan
+luc Aringa
+hyw Armenian, Western
+apr Arop-Lokep
+aia Arosi
+msy Aruamu
+cni Asháninka
+cjo Ashéninka, Pajonal
+cpu Ashéninka, Pichis
+cpb Ashéninka, Ucayali-Yurúa
+asm Assamese
+asa Asu
+teo Ateso
+ati Attié
+djk Aukan
+ava Avar
+avn Avatime
+avu Avokaya
+awb Awa
+kwi Awa-Cuaiquer
+awa Awadhi
+agr Awajún
+agu Awakateko
+ayr Aymara, Central
+ayo Ayoreo
+abp Ayta, Abellen
+blx Ayta, Mag-Indi
+sgb Ayta, Mag-antsi
+azj-script_cyrillic Azerbaijani, North
+azj-script_latin Azerbaijani, North
+azb Azerbaijani, South
+bba Baatonum
+bhz Bada
+bvc Baelelea
+bfy Bagheli
+bgq Bagri
+bdq Bahnar
+bdh Baka
+bqi Bakhtiâri
+bjw Bakwé
+blz Balantak
+ban Bali
+bcc-script_latin Balochi, Southern
+bcc-script_arabic Balochi, Southern
+bam Bamanankan
+ptu Bambam
+bcw Bana
+bqj Bandial
+bno Bantoanon
+bbb Barai
+bfa Bari
+bjz Baruga
+bak Bashkort
+eus Basque
+bsq Bassa
+akb Batak Angkola
+btd Batak Dairi
+btx Batak Karo
+bts Batak Simalungun
+bbc Batak Toba
+bvz Bauzi
+bjv Bedjond
+bep Behoa
+bkv Bekwarra
+bzj Belize English Creole
+bem Bemba
+bng Benga
+ben Bengali
+bom Berom
+btt Bete-Bendi
+bha Bharia
+bgw Bhatri
+bht Bhattiyali
+beh Biali
+sne Bidayuh, Bau
+ubl Bikol, Buhi’non
+bcl Bikol, Central
+bim Bimoba
+bkd Binukid
+bjr Binumarien
+bfo Birifor, Malba
+biv Birifor, Southern
+bib Bisa
+bis Bislama
+bzi Bisu
+bqp Bisã
+bpr Blaan, Koronadal
+bps Blaan, Sarangani
+bwq Bobo Madaré, Southern
+bdv Bodo Parja
+bqc Boko
+bus Bokobaru
+bnp Bola
+bmq Bomu
+bdg Bonggi
+boa Bora
+ksr Borong
+bor Borôro
+bru Bru, Eastern
+box Buamu
+bzh Buang, Mapos
+bgt Bughotu
+sab Buglere
+bul Bulgarian
+bwu Buli
+bmv Bum
+mya Burmese
+tte Bwanabwana
+cjp Cabécar
+cbv Cacua
+kaq Capanahua
+cot Caquinte
+cbc Carapana
+car Carib
+cat Catalan
+ceb Cebuano
+cme Cerma
+cbi Chachi
+ceg Chamacoco
+cly Chatino, Eastern Highland
+cya Chatino, Nopala
+che Chechen
+hne Chhattisgarhi
+nya Chichewa
+dig Chidigo
+dug Chiduruma
+bgr Chin, Bawm
+cek Chin, Eastern Khumi
+cfm Chin, Falam
+cnh Chin, Hakha
+hlt Chin, Matu
+mwq Chin, Müün
+ctd Chin, Tedim
+tcz Chin, Thado
+zyp Chin, Zyphe
+cco Chinantec, Comaltepec
+cnl Chinantec, Lalana
+cle Chinantec, Lealao
+chz Chinantec, Ozumacín
+cpa Chinantec, Palantla
+cso Chinantec, Sochiapam
+cnt Chinantec, Tepetotutla
+cuc Chinantec, Usila
+hak Chinese, Hakka
+nan Chinese, Min Nan
+xnj Chingoni
+cap Chipaya
+cax Chiquitano
+ctg Chittagonian
+ctu Chol
+chf Chontal, Tabasco
+cce Chopi
+crt Chorote, Iyojwa’ja
+crq Chorote, Iyo’wujwa
+cac-dialect_sansebastiáncoatán Chuj
+cac-dialect_sanmateoixtatán Chuj
+ckt Chukchi
+ncu Chumburung
+cdj Churahi
+chv Chuvash
+caa Ch’orti’
+asg Cishingini
+con Cofán
+crn Cora, El Nayar
+cok Cora, Santa Teresa
+crk-script_latin Cree, Plains
+crk-script_syllabics Cree, Plains
+crh Crimean Tatar
+cui Cuiba
+dsh Daasanach
+dbq Daba
+dga Dagaare, Southern
+dgi Dagara, Northern
+dgk Dagba
+dnj-dialect_gweetaawueast Dan
+dnj-dialect_blowowest Dan
+daa Dangaléat
+dnt Dani, Mid Grand Valley
+dnw Dani, Western
+dar Dargwa
+tcc Datooga
+dwr Dawro
+ded Dedua
+mzw Deg
+ntr Delo
+ddn Dendi
+des Desano
+dso Desiya
+nfa Dhao
+dhi Dhimal
+gud Dida, Yocoboué
+did Didinga
+mhu Digaro-Mishmi
+dip Dinka, Northeastern
+dik Dinka, Southwestern
+tbz Ditammari
+dts Dogon, Toro So
+dos Dogosé
+dgo Dogri
+mvp Duri
+nld Dutch
+jen Dza
+dzo Dzongkha
+idd Ede Idaca
+eka Ekajuk
+cto Embera Catío
+emp Emberá, Northern
+eng English
+enx Enxet
+sja Epena
+myv Erzya
+mcq Ese
+ese Ese Ejja
+evn Evenki
+eza Ezaa
+fal Fali, South
+fao Faroese
+far Fataleka
+fij Fijian
+fin Finnish
+fon Fon
+frd Fordata
+fra French
+ful Fulah
+flr Fuliiru
+gau Gadaba, Mudhili
+gbk Gaddi
+gag-script_cyrillic Gagauz
+gag-script_latin Gagauz
+gbi Galela
+gmv Gamo
+lug Ganda
+pwg Gapapaiwa
+gbm Garhwali
+cab Garifuna
+grt Garo
+krs Gbaya
+gso Gbaya, Southwest
+nlg Gela
+gej Gen
+deu German, Standard
+gri Ghari
+kik Gikuyu
+acd Gikyode
+glk Gilaki
+gof-script_latin Gofa
+gog Gogo
+gkn Gokana
+wsg Gondi, Adilabad
+gjn Gonja
+gqr Gor
+gor Gorontalo
+gux Gourmanchéma
+gbo Grebo, Northern
+ell Greek
+grc Greek, Ancient
+guh Guahibo
+gub Guajajára
+grn Guarani
+gyr Guarayu
+guo Guayabero
+gde Gude
+guj Gujarati
+gvl Gulay
+guk Gumuz
+rub Gungu
+dah Gwahatike
+gwr Gwere
+gwi Gwich’in
+hat Haitian Creole
+hlb Halbi
+amf Hamer-Banna
+hag Hanga
+hnn Hanunoo
+bgc Haryanvi
+had Hatam
+hau Hausa
+hwc Hawaii Pidgin
+hvn Hawu
+hay Haya
+xed Hdi
+heb Hebrew
+heh Hehe
+hil Hiligaynon
+hin Hindi
+hif Hindi, Fiji
+hns Hindustani, Sarnami
+hoc Ho
+hoy Holiya
+hus-dialect_westernpotosino Huastec
+hus-dialect_centralveracruz Huastec
+huv Huave, San Mateo del Mar
+hui Huli
+hun Hungarian
+hap Hupla
+iba Iban
+isl Icelandic
+dbj Ida’an
+ifa Ifugao, Amganad
+ifb Ifugao, Batad
+ifu Ifugao, Mayoyao
+ifk Ifugao, Tuwali
+ife Ifè
+ign Ignaciano
+ikk Ika
+iqw Ikwo
+ilb Ila
+ilo Ilocano
+imo Imbongu
+ind Indonesian
+inb Inga
+ipi Ipili
+irk Iraqw
+icr Islander English Creole
+itv Itawit
+itl Itelmen
+atg Ivbie North-Okpela-Arhe
+ixl-dialect_sanjuancotzal Ixil
+ixl-dialect_sangasparchajul Ixil
+ixl-dialect_santamarianebaj Ixil
+nca Iyo
+izr Izere
+izz Izii
+jac Jakalteko
+jam Jamaican English Creole
+jav Javanese
+jvn Javanese, Suriname
+kac Jingpho
+dyo Jola-Fonyi
+csk Jola-Kasa
+adh Jopadhola
+jun Juang
+jbu Jukun Takum
+dyu Jula
+bex Jur Modo
+juy Juray
+gna Kaansa
+urb Kaapor
+kbp Kabiyè
+cwa Kabwa
+dtp Kadazan Dusun
+kbr Kafa
+cgc Kagayanen
+kki Kagulu
+kzf Kaili, Da’a
+lew Kaili, Ledo
+cbr Kakataibo-Kashibo
+kkj Kako
+keo Kakwa
+kqe Kalagan
+kak Kalanguya
+kyb Kalinga, Butbut
+knb Kalinga, Lubuagan
+kmd Kalinga, Majukayang
+kml Kalinga, Tanudan
+ify Kallahan, Keley-i
+xal Kalmyk-Oirat
+kbq Kamano
+kay Kamayurá
+ktb Kambaata
+hig Kamwe
+gam Kandawo
+cbu Kandozi-Chapra
+xnr Kangri
+kmu Kanite
+kne Kankanaey
+kan Kannada
+kby Kanuri, Manga
+pam Kapampangan
+cak-dialect_santamaríadejesús Kaqchikel
+cak-dialect_southcentral Kaqchikel
+cak-dialect_yepocapa Kaqchikel
+cak-dialect_western Kaqchikel
+cak-dialect_santodomingoxenacoj Kaqchikel
+cak-dialect_central Kaqchikel
+xrb Karaboro, Eastern
+krc Karachay-Balkar
+kaa Karakalpak
+krl Karelian
+pww Karen, Pwo Northern
+xsm Kasem
+cbs Kashinawa
+pss Kaulong
+kxf Kawyaw
+kyz Kayabí
+kyu Kayah, Western
+txu Kayapó
+kaz Kazakh
+ndp Kebu
+kbo Keliko
+kyq Kenga
+ken Kenyang
+ker Kera
+xte Ketengban
+kyg Keyagana
+kjh Khakas
+kca Khanty
+khm Khmer
+kxm Khmer, Northern
+kjg Khmu
+nyf Kigiryama
+kij Kilivila
+kia Kim
+kqr Kimaragang
+kqp Kimré
+krj Kinaray-a
+zga Kinga
+kin Kinyarwanda
+pkb Kipfokomo
+geb Kire
+gil Kiribati
+kje Kisar
+kss Kisi, Southern
+thk Kitharaka
+klu Klao
+kyo Klon
+kog Kogi
+kfb Kolami, Northwestern
+kpv Komi-Zyrian
+bbo Konabéré
+xon Konkomba
+kma Konni
+kno Kono
+kxc Konso
+ozm Koonzime
+kqy Koorete
+kor Korean
+coe Koreguaje
+kpq Korupun-Sela
+kpy Koryak
+kyf Kouya
+kff-script_telugu Koya
+kri Krio
+rop Kriol
+ktj Krumen, Plapo
+ted Krumen, Tepo
+krr Krung
+kdt Kuay
+kez Kukele
+cul Kulina
+kle Kulung
+kdi Kumam
+kue Kuman
+kum Kumyk
+kvn Kuna, Border
+cuk Kuna, San Blas
+kdn Kunda
+xuo Kuo
+key Kupia
+kpz Kupsapiiny
+knk Kuranko
+kmr-script_latin Kurdish, Northern
+kmr-script_arabic Kurdish, Northern
+kmr-script_cyrillic Kurdish, Northern
+xua Kurumba, Alu
+kru Kurux
+kus Kusaal
+kub Kutep
+kdc Kutu
+kxv Kuvi
+blh Kuwaa
+cwt Kuwaataay
+kwd Kwaio
+tnk Kwamera
+kwf Kwara’ae
+cwe Kwere
+kyc Kyaka
+tye Kyanga
+kir Kyrgyz
+quc-dialect_north K’iche’
+quc-dialect_east K’iche’
+quc-dialect_central K’iche’
+lac Lacandon
+lsi Lacid
+lbj Ladakhi
+lhu Lahu
+las Lama
+lam Lamba
+lns Lamnso’
+ljp Lampung Api
+laj Lango
+lao Lao
+lat Latin
+lav Latvian
+law Lauje
+lcp Lawa, Western
+lzz Laz
+lln Lele
+lef Lelemi
+acf Lesser Antillean French Creole
+lww Lewo
+mhx Lhao Vo
+eip Lik
+lia Limba, West-Central
+lif Limbu
+onb Lingao
+lis Lisu
+loq Lobala
+lob Lobi
+yaz Lokaa
+lok Loko
+llg Lole
+ycl Lolopo
+lom Loma
+ngl Lomwe
+lon Lomwe, Malawi
+lex Luang
+lgg Lugbara
+ruf Luguru
+dop Lukpa
+lnd Lundayeh
+ndy Lutos
+lwo Luwo
+lee Lyélé
+mev Maan
+mfz Mabaan
+jmc Machame
+myy Macuna
+mbc Macushi
+mda Mada
+mad Madura
+mag Magahi
+ayz Mai Brat
+mai Maithili
+mca Maka
+mcp Makaa
+mak Makasar
+vmw Makhuwa
+mgh Makhuwa-Meetto
+kde Makonde
+mlg Malagasy
+zlm Malay
+pse Malay, Central
+mkn Malay, Kupang
+xmm Malay, Manado
+mal Malayalam
+xdy Malayic Dayak
+div Maldivian
+mdy Male
+mup Malvi
+mam-dialect_central Mam
+mam-dialect_northern Mam
+mam-dialect_southern Mam
+mam-dialect_western Mam
+mqj Mamasa
+mcu Mambila, Cameroon
+mzk Mambila, Nigeria
+maw Mampruli
+mjl Mandeali
+mnk Mandinka
+mge Mango
+mbh Mangseng
+knf Mankanya
+mjv Mannan
+mbt Manobo, Matigsalug
+obo Manobo, Obo
+mbb Manobo, Western Bukidnon
+mzj Manya
+sjm Mapun
+mrw Maranao
+mar Marathi
+mpg Marba
+mhr Mari, Meadow
+enb Markweeta
+mah Marshallese
+myx Masaaba
+klv Maskelynes
+mfh Matal
+met Mato
+mcb Matsigenka
+mop Maya, Mopán
+yua Maya, Yucatec
+mfy Mayo
+maz Mazahua, Central
+vmy Mazatec, Ayautla
+maq Mazatec, Chiquihuitlán
+mzi Mazatec, Ixcatlán
+maj Mazatec, Jalapa de Díaz
+maa-dialect_sanantonio Mazatec, San Jerónimo Tecóatl
+maa-dialect_sanjerónimo Mazatec, San Jerónimo Tecóatl
+mhy Ma’anyan
+mhi Ma’di
+zmz Mbandja
+myb Mbay
+gai Mbore
+mqb Mbuko
+mbu Mbula-Bwazza
+med Melpa
+men Mende
+mee Mengen
+mwv Mentawai
+meq Merey
+zim Mesme
+mgo Meta’
+mej Meyah
+mpp Migabac
+min Minangkabau
+gum Misak
+mpx Misima-Panaeati
+mco Mixe, Coatlán
+mxq Mixe, Juquila
+pxm Mixe, Quetzaltepec
+mto Mixe, Totontepec
+mim Mixtec, Alacatlatzala
+xta Mixtec, Alcozauca
+mbz Mixtec, Amoltepec
+mip Mixtec, Apasco-Apoala
+mib Mixtec, Atatlahuca
+miy Mixtec, Ayutla
+mih Mixtec, Chayuco
+miz Mixtec, Coatzospan
+xtd Mixtec, Diuxi-Tilantongo
+mxt Mixtec, Jamiltepec
+xtm Mixtec, Magdalena Peñasco
+mxv Mixtec, Metlatónoc
+xtn Mixtec, Northern Tlaxiaco
+mie Mixtec, Ocotepec
+mil Mixtec, Peñoles
+mio Mixtec, Pinotepa Nacional
+mdv Mixtec, Santa Lucía Monteverde
+mza Mixtec, Santa María Zacatepec
+mit Mixtec, Southern Puebla
+mxb Mixtec, Tezoatlán
+mpm Mixtec, Yosondúa
+soy Miyobe
+cmo-script_latin Mnong, Central
+cmo-script_khmer Mnong, Central
+mfq Moba
+old Mochi
+mfk Mofu, North
+mif Mofu-Gudur
+mkl Mokole
+mox Molima
+myl Moma
+mqf Momuna
+mnw Mon
+mon Mongolian
+mog Mongondow
+mfe Morisyen
+mor Moro
+mqn Moronene
+mgd Moru
+mtj Moskona
+cmr Mro-Khimi
+mtd Mualang
+bmr Muinane
+moz Mukulu
+mzm Mumuye
+mnb Muna
+mnf Mundani
+unr Mundari
+fmu Muria, Far Western
+mur Murle
+tih Murut, Timugon
+muv Muthuvan
+muy Muyang
+sur Mwaghavul
+moa Mwan
+wmw Mwani
+tnr Ménik
+miq Mískito
+mos Mòoré
+muh Mündü
+nas Naasioi
+mbj Nadëb
+nfr Nafaanra
+kfw Naga, Kharam
+nst Naga, Tangshang
+nag Nagamese
+nch Nahuatl, Central Huasteca
+nhe Nahuatl, Eastern Huasteca
+ngu Nahuatl, Guerrero
+azz Nahuatl, Highland Puebla
+nhx Nahuatl, Isthmus-Mecayapan
+ncl Nahuatl, Michoacán
+nhy Nahuatl, Northern Oaxaca
+ncj Nahuatl, Northern Puebla
+nsu Nahuatl, Sierra Negra
+npl Nahuatl, Southeastern Puebla
+nuz Nahuatl, Tlamacazapa
+nhw Nahuatl, Western Huasteca
+nhi Nahuatl, Zacatlán-Ahuacatlán-Tepetzintla
+nlc Nalca
+nab Nambikuára, Southern
+gld Nanai
+nnb Nande
+npy Napu
+pbb Nasa
+ntm Nateni
+nmz Nawdm
+naw Nawuri
+nxq Naxi
+ndj Ndamba
+ndz Ndogo
+ndv Ndut
+new Newar
+nij Ngaju
+sba Ngambay
+gng Ngangam
+nga Ngbaka
+nnq Ngindo
+ngp Ngulu
+gym Ngäbere
+kdj Ng’akarimojong
+nia Nias
+nim Nilamba
+nin Ninzo
+nko Nkonya
+nog Nogai
+lem Nomaande
+not Nomatsigenga
+nhu Noone
+bud Ntcham
+nus Nuer
+yas Nugunu
+nnw Nuni, Southern
+nwb Nyabwa
+nyy Nyakyusa-Ngonde
+nyn Nyankore
+rim Nyaturu
+lid Nyindrou
+nuj Nyole
+nyo Nyoro
+nzi Nzema
+ann Obolo
+ory Odia
+ojb-script_latin Ojibwa, Northwestern
+ojb-script_syllabics Ojibwa, Northwestern
+oku Oku
+bsc Oniyan
+bdu Oroko
+orm Oromo
+ury Orya
+oss Ossetic
+ote Otomi, Mezquital
+otq Otomi, Querétaro
+stn Owa
+sig Paasaal
+kfx Pahari, Kullu
+bfz Pahari, Mahasu
+sey Paicoca
+pao Paiute, Northern
+pau Palauan
+pce Palaung, Ruching
+plw Palawano, Brooke’s Point
+pmf Pamona
+pag Pangasinan
+pap Papiamentu
+prf Paranan
+pab Parecís
+pbi Parkwa
+pbc Patamona
+pad Paumarí
+ata Pele-Ata
+pez Penan, Eastern
+peg Pengo
+fas Persian
+pcm Pidgin, Nigerian
+pis Pijin
+pny Pinyin
+pir Piratapuyo
+pjt Pitjantjatjara
+poy Pogolo
+pol Polish
+pps Popoloca, San Luís Temalacayuca
+pls Popoloca, San Marcos Tlacoyalco
+poi Popoluca, Highland
+poh-dialect_eastern Poqomchi’
+poh-dialect_western Poqomchi’
+por Portuguese
+prt Prai
+pui Puinave
+pan Punjabi, Eastern
+tsz Purepecha
+suv Puroik
+lme Pévé
+quy Quechua, Ayacucho
+qvc Quechua, Cajamarca
+quz Quechua, Cusco
+qve Quechua, Eastern Apurímac
+qub Quechua, Huallaga
+qvh Quechua, Huamalíes-Dos de Mayo Huánuco
+qwh Quechua, Huaylas Ancash
+qvw Quechua, Huaylla Wanca
+quf Quechua, Lambayeque
+qvm Quechua, Margos-Yarowilca-Lauricocha
+qul Quechua, North Bolivian
+qvn Quechua, North Junín
+qxn Quechua, Northern Conchucos Ancash
+qxh Quechua, Panao
+qvs Quechua, San Martín
+quh Quechua, South Bolivian
+qxo Quechua, Southern Conchucos
+qxr Quichua, Cañar Highland
+qvo Quichua, Napo
+qvz Quichua, Northern Pastaza
+qxl Quichua, Salasaca Highland
+quw Quichua, Tena Lowland
+kjb Q’anjob’al
+kek Q’eqchi’
+rah Rabha
+rjs Rajbanshi
+rai Ramoaaina
+lje Rampi
+rnl Ranglong
+rkt Rangpuri
+rap Rapa Nui
+yea Ravula
+raw Rawang
+rej Rejang
+rel Rendille
+ril Riang Lang
+iri Rigwe
+rgu Rikou
+rhg Rohingya
+rmc-script_latin Romani, Carpathian
+rmc-script_cyrillic Romani, Carpathian
+rmo Romani, Sinte
+rmy-script_latin Romani, Vlax
+rmy-script_cyrillic Romani, Vlax
+ron Romanian
+rol Romblomanon
+cla Ron
+rng Ronga
+rug Roviana
+run Rundi
+rus Russian
+lsm Saamya-Gwe
+spy Sabaot
+sck Sadri
+saj Sahu
+sch Sakachep
+sml Sama, Central
+xsb Sambal
+sbl Sambal, Botolan
+saq Samburu
+sbd Samo, Southern
+smo Samoan
+rav Sampang
+sxn Sangir
+sag Sango
+sbp Sangu
+xsu Sanumá
+srm Saramaccan
+sas Sasak
+apb Sa’a
+sgw Sebat Bet Gurage
+tvw Sedoa
+lip Sekpele
+slu Selaru
+snw Selee
+sea Semai
+sza Semelai
+seh Sena
+crs Seychelles French Creole
+ksb Shambala
+shn Shan
+sho Shanga
+mcd Sharanahua
+cbt Shawi
+xsr Sherpa
+shk Shilluk
+shp Shipibo-Conibo
+sna Shona
+cjs Shor
+jiv Shuar
+snp Siane
+sya Siang
+sid Sidamo
+snn Siona
+sri Siriano
+srx Sirmauri
+sil Sisaala, Tumulung
+sld Sissala
+akp Siwu
+xog Soga
+som Somali
+bmu Somba-Siawari
+khq Songhay, Koyra Chiini
+ses Songhay, Koyraboro Senni
+mnx Sougb
+spa Spanish
+srn Sranan Tongo
+sxb Suba
+suc Subanon, Western
+tgo Sudest
+suk Sukuma
+sun Sunda
+suz Sunwar
+sgj Surgujia
+sus Susu
+swh Swahili
+swe Swedish
+syl Sylheti
+dyi Sénoufo, Djimini
+myk Sénoufo, Mamara
+spp Sénoufo, Supyire
+tap Taabwa
+tby Tabaru
+tna Tacana
+shi Tachelhit
+klw Tado
+tgl Tagalog
+tbk Tagbanwa, Calamian
+tgj Tagin
+blt Tai Dam
+tbg Tairora, North
+omw Tairora, South
+tgk Tajik
+tdj Tajio
+tbc Takia
+tlj Talinga-Bwisi
+tly Talysh
+ttq-script_tifinagh Tamajaq, Tawallammat
+taj Tamang, Eastern
+taq Tamasheq
+tam Tamil
+tpm Tampulma
+tgp Tangoa
+tnn Tanna, North
+tac Tarahumara, Western
+rif-script_latin Tarifit
+rif-script_arabic Tarifit
+tat Tatar
+tav Tatuyo
+twb Tawbuid
+tbl Tboli
+kps Tehit
+twe Teiwa
+ttc Tektiteko
+tel Telugu
+kdh Tem
+tes Tengger
+tex Tennet
+tee Tepehua, Huehuetla
+tpp Tepehua, Pisaflores
+tpt Tepehua, Tlachichilco
+stp Tepehuan, Southeastern
+tfr Teribe
+twu Termanu
+ter Terêna
+tew Tewa
+tha Thai
+nod Thai, Northern
+thl Tharu, Dangaura
+tem Themne
+adx Tibetan, Amdo
+bod Tibetan, Central
+khg Tibetan, Khams
+tca Ticuna
+tir Tigrigna
+txq Tii
+tik Tikar
+dgr Tlicho
+tob Toba
+tmf Toba-Maskoy
+tng Tobanga
+tlb Tobelo
+ood Tohono O’odham
+tpi Tok Pisin
+jic Tol
+lbw Tolaki
+txa Tombonuo
+tom Tombulu
+toh Tonga
+tnt Tontemboan
+sda Toraja-Sa’dan
+tcs Torres Strait Creole
+toc Totonac, Coyutla
+tos Totonac, Highland
+neb Toura
+trn Trinitario
+trs Triqui, Chicahuaxtla
+trc Triqui, Copala
+tri Trió
+cof Tsafiki
+tkr Tsakhur
+kdl Tsikimba
+cas Tsimané
+tso Tsonga
+tuo Tucano
+iou Tuma-Irumu
+tmc Tumak
+tuf Tunebo, Central
+tur Turkish
+tuk-script_latin Turkmen
+tuk-script_arabic Turkmen
+bov Tuwuli
+tue Tuyuca
+kcg Tyap
+tzh-dialect_bachajón Tzeltal
+tzh-dialect_tenejapa Tzeltal
+tzo-dialect_chenalhó Tzotzil
+tzo-dialect_chamula Tzotzil
+tzj-dialect_western Tz’utujil
+tzj-dialect_eastern Tz’utujil
+aoz Uab Meto
+udm Udmurt
+udu Uduk
+ukr Ukrainian
+ppk Uma
+ubu Umbu-Ungu
+urk Urak Lawoi’
+ura Urarina
+urt Urat
+urd-script_devanagari Urdu
+urd-script_arabic Urdu
+urd-script_latin Urdu
+upv Uripiv-Wala-Rano-Atchin
+usp Uspanteko
+uig-script_arabic Uyghur
+uig-script_cyrillic Uyghur
+uzb-script_cyrillic Uzbek
+vag Vagla
+bav Vengo
+vid Vidunda
+vie Vietnamese
+vif Vili
+vun Vunjo
+vut Vute
+prk Wa, Parauk
+wwa Waama
+rro Waima
+bao Waimaha
+waw Waiwai
+lgl Wala
+wlx Wali
+cou Wamey
+hub Wampís
+gvc Wanano
+mfi Wandala
+wap Wapishana
+wba Warao
+war Waray-Waray
+way Wayana
+guc Wayuu
+cym Welsh
+kvw Wersing
+tnp Whitesands
+hto Witoto, Minika
+huu Witoto, Murui
+wal-script_latin Wolaytta
+wal-script_ethiopic Wolaytta
+wlo Wolio
+noa Woun Meu
+wob Wè Northern
+kao Xaasongaxango
+xer Xerénte
+yad Yagua
+yka Yakan
+sah Yakut
+yba Yala
+yli Yali, Angguruk
+nlk Yali, Ninia
+yal Yalunka
+yam Yamba
+yat Yambeta
+jmd Yamdena
+tao Yami
+yaa Yaminahua
+ame Yanesha’
+guu Yanomamö
+yao Yao
+yre Yaouré
+yva Yawa
+ybb Yemba
+pib Yine
+byr Yipma
+pil Yom
+yor Yoruba
+ycn Yucuna
+ess Yupik, Saint Lawrence Island
+yuz Yuracare
+atb Zaiwa
+zne Zande
+zaq Zapotec, Aloápam
+zpo Zapotec, Amatlán
+zad Zapotec, Cajonos
+zpc Zapotec, Choapan
+zca Zapotec, Coatecas Altas
+zpg Zapotec, Guevea de Humboldt
+zai Zapotec, Isthmus
+zpl Zapotec, Lachixío
+zam Zapotec, Miahuatlán
+zaw Zapotec, Mitla
+zpm Zapotec, Mixtepec
+zac Zapotec, Ocotlán
+zao Zapotec, Ozolotepec
+ztq Zapotec, Quioquitani-Quierí
+zar Zapotec, Rincón
+zpt Zapotec, San Vicente Coatlán
+zpi Zapotec, Santa María Quiegolani
+zas Zapotec, Santo Domingo Albarradas
+zaa Zapotec, Sierra de Juárez
+zpz Zapotec, Texmelucan
+zab Zapotec, Western Tlacolula Valley
+zpu Zapotec, Yalálag
+zae Zapotec, Yareni
+zty Zapotec, Yatee
+zav Zapotec, Yatzachi
+zza Zaza
+zyb Zhuang, Yongbei
+ziw Zigula
+zos Zoque, Francisco León
+gnd Zulgo-Gemzek
+ewe Éwé

lid.py

@@ -0,0 +1,70 @@
+from transformers import Wav2Vec2ForSequenceClassification, AutoFeatureExtractor
+import torch
+import librosa
+
+model_id = "facebook/mms-lid-1024"
+
+processor = AutoFeatureExtractor.from_pretrained(model_id)
+model = Wav2Vec2ForSequenceClassification.from_pretrained(model_id)
+
+
+LID_SAMPLING_RATE = 16_000
+LID_TOPK = 10
+LID_THRESHOLD = 0.33
+
+LID_LANGUAGES = {}
+with open(f"data/lid/all_langs.tsv") as f:
+    for line in f:
+        iso, name = line.split(" ", 1)
+        LID_LANGUAGES[iso] = name
+
+
+def identify(audio_source=None, microphone=None, file_upload=None):
+    if audio_source is None and microphone is None and file_upload is None:
+        # HACK: need to handle this case for some reason
+        return {}
+
+    if type(microphone) is dict:
+        # HACK: microphone variable is a dict when running on examples
+        microphone = microphone["name"]
+    audio_fp = (
+        file_upload if "upload" in str(audio_source or "").lower() else microphone
+    )
+    audio_samples = librosa.load(audio_fp, sr=LID_SAMPLING_RATE, mono=True)[0]
+
+    inputs = processor(
+        audio_samples, sampling_rate=LID_SAMPLING_RATE, return_tensors="pt"
+    )
+
+    # set device
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    elif (
+        hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+        and torch.backends.mps.is_built()
+    ):
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+
+    model.to(device)
+    inputs = inputs.to(device)
+
+    with torch.no_grad():
+        logit = model(**inputs).logits
+
+    logit_lsm = torch.log_softmax(logit.squeeze(), dim=-1)
+    scores, indices = torch.topk(logit_lsm, 5, dim=-1)
+    scores, indices = torch.exp(scores).to("cpu").tolist(), indices.to("cpu").tolist()
+    iso2score = {model.config.id2label[int(i)]: s for s, i in zip(scores, indices)}
+    if max(iso2score.values()) < LID_THRESHOLD:
+        return "Low confidence in the language identification predictions. Output is not shown!"
+    return {LID_LANGUAGES[iso]: score for iso, score in iso2score.items()}
+
+
+LID_EXAMPLES = [
+    [None, "./assets/english.mp3", None],
+    [None, "./assets/tamil.mp3", None],
+    [None, "./assets/burmese.mp3", None],
+]

requirements.txt

@@ -0,0 +1,12 @@
+accelerate
+librosa
+git+https://github.com/huggingface/transformers.git
+torch
+Cython==0.29.21
+phonemizer==2.2.1
+scipy
+numpy
+torchaudio
+matplotlib
+Unidecode==1.1.1
+monotonic-align

tts.py

@@ -0,0 +1,179 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import re
+import tempfile
+import torch
+import sys
+import gradio as gr
+
+from huggingface_hub import hf_hub_download
+
+# Setup TTS env
+if "vits" not in sys.path:
+    sys.path.append("vits")
+
+from vits import commons, utils
+from vits.models import SynthesizerTrn
+
+
+TTS_LANGUAGES = {}
+with open(f"data/tts/all_langs.tsv") as f:
+    for line in f:
+        iso, name = line.split(" ", 1)
+        TTS_LANGUAGES[iso] = name
+
+
+class TextMapper(object):
+    def __init__(self, vocab_file):
+        self.symbols = [
+            x.replace("\n", "") for x in open(vocab_file, encoding="utf-8").readlines()
+        ]
+        self.SPACE_ID = self.symbols.index(" ")
+        self._symbol_to_id = {s: i for i, s in enumerate(self.symbols)}
+        self._id_to_symbol = {i: s for i, s in enumerate(self.symbols)}
+
+    def text_to_sequence(self, text, cleaner_names):
+        """Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+        Args:
+        text: string to convert to a sequence
+        cleaner_names: names of the cleaner functions to run the text through
+        Returns:
+        List of integers corresponding to the symbols in the text
+        """
+        sequence = []
+        clean_text = text.strip()
+        for symbol in clean_text:
+            symbol_id = self._symbol_to_id[symbol]
+            sequence += [symbol_id]
+        return sequence
+
+    def uromanize(self, text, uroman_pl):
+        iso = "xxx"
+        with tempfile.NamedTemporaryFile() as tf, tempfile.NamedTemporaryFile() as tf2:
+            with open(tf.name, "w") as f:
+                f.write("\n".join([text]))
+            cmd = f"perl " + uroman_pl
+            cmd += f" -l {iso} "
+            cmd += f" < {tf.name} > {tf2.name}"
+            os.system(cmd)
+            outtexts = []
+            with open(tf2.name) as f:
+                for line in f:
+                    line = re.sub(r"\s+", " ", line).strip()
+                    outtexts.append(line)
+            outtext = outtexts[0]
+        return outtext
+
+    def get_text(self, text, hps):
+        text_norm = self.text_to_sequence(text, hps.data.text_cleaners)
+        if hps.data.add_blank:
+            text_norm = commons.intersperse(text_norm, 0)
+        text_norm = torch.LongTensor(text_norm)
+        return text_norm
+
+    def filter_oov(self, text, lang=None):
+        text = self.preprocess_char(text, lang=lang)
+        val_chars = self._symbol_to_id
+        txt_filt = "".join(list(filter(lambda x: x in val_chars, text)))
+        return txt_filt
+
+    def preprocess_char(self, text, lang=None):
+        """
+        Special treatement of characters in certain languages
+        """
+        if lang == "ron":
+            text = text.replace("ț", "ţ")
+            print(f"{lang} (ț -> ţ): {text}")
+        return text
+
+
+def synthesize(text, lang, speed=None):
+    if speed is None:
+        speed = 1.0
+
+    lang_code = lang.split()[0].strip()
+
+    vocab_file = hf_hub_download(
+        repo_id="facebook/mms-tts",
+        filename="vocab.txt",
+        subfolder=f"models/{lang_code}",
+    )
+    config_file = hf_hub_download(
+        repo_id="facebook/mms-tts",
+        filename="config.json",
+        subfolder=f"models/{lang_code}",
+    )
+    g_pth = hf_hub_download(
+        repo_id="facebook/mms-tts",
+        filename="G_100000.pth",
+        subfolder=f"models/{lang_code}",
+    )
+
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+    elif (
+        hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+        and torch.backends.mps.is_built()
+    ):
+        device = torch.device("mps")
+    else:
+        device = torch.device("cpu")
+
+    print(f"Run inference with {device}")
+
+    assert os.path.isfile(config_file), f"{config_file} doesn't exist"
+    hps = utils.get_hparams_from_file(config_file)
+    text_mapper = TextMapper(vocab_file)
+    net_g = SynthesizerTrn(
+        len(text_mapper.symbols),
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        **hps.model,
+    )
+    net_g.to(device)
+    _ = net_g.eval()
+
+    _ = utils.load_checkpoint(g_pth, net_g, None)
+
+    is_uroman = hps.data.training_files.split(".")[-1] == "uroman"
+
+    if is_uroman:
+        uroman_dir = "uroman"
+        assert os.path.exists(uroman_dir)
+        uroman_pl = os.path.join(uroman_dir, "bin", "uroman.pl")
+        text = text_mapper.uromanize(text, uroman_pl)
+
+    text = text.lower()
+    text = text_mapper.filter_oov(text, lang=lang)
+    stn_tst = text_mapper.get_text(text, hps)
+    with torch.no_grad():
+        x_tst = stn_tst.unsqueeze(0).to(device)
+        x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(device)
+        hyp = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                noise_scale=0.667,
+                noise_scale_w=0.8,
+                length_scale=1.0 / speed,
+            )[0][0, 0]
+            .cpu()
+            .float()
+            .numpy()
+        )
+
+    return gr.Audio.update(value=(hps.data.sampling_rate, hyp)), text
+
+
+TTS_EXAMPLES = [
+    ["I am going to the store.", "eng (English)"],
+    ["안녕하세요", "kor (Korean)"],
+    ["क्या मुझे पीने का पानी मिल सकता है?", "hin (Hindi)"],
+    ["Tanış olmağıma çox şadam", "azj-script_latin (Azerbaijani, North)"],
+    ["Mu zo murna a cikin ƙasar.", "hau (Hausa)"],
+]

uroman/.gitignore

@@ -0,0 +1,35 @@
+!Build/
+.last_cover_stats
+/META.yml
+/META.json
+/MYMETA.*
+*.o
+*.pm.tdy
+*.bs
+
+# Devel::Cover
+cover_db/
+
+# Devel::NYTProf
+nytprof.out
+
+# Dizt::Zilla
+/.build/
+
+# Module::Build
+_build/
+Build
+Build.bat
+
+# Module::Install
+inc/
+
+# ExtUtils::MakeMaker
+/blib/
+/_eumm/
+/*.gz
+/Makefile
+/Makefile.old
+/MANIFEST.bak
+/pm_to_blib
+/*.zip

uroman/LICENSE.txt

@@ -0,0 +1,11 @@
+Copyright (C) 2015-2020 Ulf Hermjakob, USC Information Sciences Institute
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+Any publication of projects using uroman shall acknowledge its use: "This project uses the universal romanizer software 'uroman' written by Ulf Hermjakob, USC Information Sciences Institute (2015-2020)".
+Bibliography: Ulf Hermjakob, Jonathan May, and Kevin Knight. 2018. Out-of-the-box universal romanization tool uroman. In Proceedings of the 56th Annual Meeting of Association for Computational Linguistics, Demo Track.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

uroman/README.md

@@ -0,0 +1,165 @@
+# uroman
+
+*uroman* is a *universal romanizer*. It converts text in any script to the Latin alphabet.
+
+Version: 1.2.8
+Release date: April 23, 2021  
+Author: Ulf Hermjakob, USC Information Sciences Institute  
+
+
+### Usage
+```bash
+$ uroman.pl [-l <lang-code>] [--chart] [--no-cache] < STDIN
+       where the optional <lang-code> is a 3-letter languages code, e.g. ara, bel, bul, deu, ell, eng, fas,
+            grc, ell, eng, heb, kaz, kir, lav, lit, mkd, mkd2, oss, pnt, pus, rus, srp, srp2, tur, uig, ukr, yid.
+       --chart specifies chart output (in JSON format) to represent alternative romanizations.
+       --no-cache disables caching.
+```
+### Examples
+```bash
+$ bin/uroman.pl < text/zho.txt
+$ bin/uroman.pl -l tur < text/tur.txt
+$ bin/uroman.pl -l heb --chart < text/heb.txt
+$ bin/uroman.pl < test/multi-script.txt > test/multi-script.uroman.txt
+```
+
+Identifying the input as Arabic, Belarusian, Bulgarian, English, Farsi, German,
+Ancient Greek, Modern Greek, Pontic Greek, Hebrew, Kazakh, Kyrgyz, Latvian,
+Lithuanian, North Macedonian, Russian, Serbian, Turkish, Ukrainian, Uyghur or 
+Yiddish will improve romanization for those languages as some letters in those 
+languages have different sound values from other languages using the same script 
+(French, Russian, Hebrew respectively).
+No effect for other languages in this version.
+
+### Bibliography
+Ulf Hermjakob, Jonathan May, and Kevin Knight. 2018. Out-of-the-box universal romanization tool uroman. In Proceedings of the 56th Annual Meeting of Association for Computational Linguistics, Demo Track. ACL-2018 Best Demo Paper Award. [Paper in ACL Anthology](https://www.aclweb.org/anthology/P18-4003) | [Poster](https://www.isi.edu/~ulf/papers/poster-uroman-acl2018.pdf) | [BibTex](https://www.aclweb.org/anthology/P18-4003.bib)
+
+### Change History
+Changes in version 1.2.8
+ * Updated to Unicode 13.0 (2021), which supports several new scripts (10% larger UnicodeData.txt).
+ * Improved support for Georgian.
+ * Preserve various symbols (as opposed to mapping to the symbols' names).
+ * Various small improvements.
+
+Changes in version 1.2.7
+ * Improved support for Pashto.
+
+Changes in version 1.2.6
+ * Improved support for Ukrainian, Russian and Ogham (ancient Irish script).
+ * Added support for English Braille.
+ * Added alternative Romanization for North Macedonian and Serbian (mkd2/srp2)
+   reflecting a casual style that many native speakers of those languages use
+   when writing text in Latin script, e.g. non-accented single letters (e.g. "s")
+   rather than phonetically motivated combinations of letters (e.g. "sh").
+ * When a line starts with "::lcode xyz ", the new uroman version will switch to
+   that language for that line. This is used for the new reference test file.
+ * Various small improvements.
+
+Changes in version 1.2.5
+ * Improved support for Armenian and eight languages using Cyrillic scripts.
+   -- For Serbian and Macedonian, which are often written in both Cyrillic
+      and Latin scripts, uroman will map both official versions to the same
+      romanized text, e.g. both "Ниш" and "Niš" will be mapped to "Nish" (which
+      properly reflects the pronunciation of the city's name).
+      For both Serbian and Macedonian, casual writers often use a simplified
+      Latin form without diacritics, e.g. "s" to represent not only Cyrillic "с"
+      and Latin "s", but also "ш" or "š", even if this conflates "s" and "sh" and
+      other such pairs. The casual romanization can be simulated by using
+      alternative uroman language codes "srp2" and "mkd2", which romanize
+      both "Ниш" and "Niš" to "Nis" to reflect the casual Latin spelling.
+ * Various small improvements.
+
+Changes in version 1.2.4
+  * Bug-fix that generated two emtpy lines for each empty line in cache mode.
+
+Changes in version 1.2
+ * Run-time improvement based on (1) token-based caching and (2) shortcut 
+   romanization (identity) of ASCII strings for default 1-best (non-chart) 
+   output. Speed-up by a factor of 10 for Bengali and Uyghur on medium and 
+   large size texts.
+ * Incremental improvements for Farsi, Amharic, Russian, Hebrew and related
+   languages.
+ * Richer lattice structure (more alternatives) for "Romanization" of English
+   to support better matching to romanizations of other languages.
+   Changes output only when --chart option is specified. No change in output for
+   default 1-best output, which for ASCII characters is always the input string.
+
+Changes in version 1.1 (major upgrade)
+ * Offers chart output (in JSON format) to represent alternative romanizations.
+   -- Location of first character is defined to be "line: 1, start:0, end:0".
+ * Incremental improvements of Hebrew and Greek romanization; Chinese numbers.
+ * Improved web-interface at http://www.isi.edu/~ulf/uroman.html
+   -- Shows corresponding original and romanization text in red
+      when hovering over a text segment.
+   -- Shows alternative romanizations when hovering over romanized text
+      marked by dotted underline.
+   -- Added right-to-left script detection and improved display for right-to-left
+      script text (as determined line by line).
+   -- On-page support for some scripts that are often not pre-installed on users'
+      computers (Burmese, Egyptian, Klingon).
+
+Changes in version 1.0 (major upgrade)
+ * Upgraded principal internal data structure from string to lattice.
+ * Improvements mostly in vowelization of South and Southeast Asian languages.
+ * Vocalic 'r' more consistently treated as vowel (no additional vowel added).
+ * Repetition signs (Japanese/Chinese/Thai/Khmer/Lao) are mapped to superscript 2.
+ * Japanese Katakana middle dots now mapped to ASCII space.
+ * Tibetan intersyllabic mark now mapped to middle dot (U+00B7).
+ * Some corrections regarding analysis of Chinese numbers.
+ * Many more foreign diacritics and punctuation marks dropped or mapped to ASCII.
+ * Zero-width characters dropped, except line/sentence-initial byte order marks.
+ * Spaces normalized to ASCII space.
+ * Fixed bug that in some cases mapped signs (such as dagger or bullet) to their verbal descriptions.
+ * Tested against previous version of uroman with a new uroman visual diff tool.
+ * Almost an order of magnitude faster.
+
+Changes in version 0.7 (minor upgrade)
+ * Added script uroman-quick.pl for Arabic script languages, incl. Uyghur.
+   Much faster, pre-caching mapping of Arabic to Latin characters, simple greedy processing.
+   Will not convert material from non-Arabic blocks such as any (somewhat unusual) Cyrillic
+   or Chinese characters in Uyghur texts.
+
+Changes in version 0.6 (minor upgrade)
+ * Added support for two letter characters used in Uzbek:
+   (1) character "ʻ" ("modifier letter turned comma", which modifies preceding "g" and "u" letters)
+   (2) character "ʼ" ("modifier letter apostrophe", which Uzbek uses to mark a glottal stop).
+   Both are now mapped to "'" (plain ASCII apostrophe).
+ * Added support for Uyghur vowel characters such as "ې" (Arabic e) and "ۆ" (Arabic oe)
+   even when they are not preceded by "ئ" (yeh with hamza above).
+ * Added support for Arabic semicolon "؛", Arabic ligature forms for phrases such as "ﷺ"
+   ("sallallahou alayhe wasallam" = "prayer of God be upon him and his family and peace")
+ * Added robustness for Arabic letter presentation forms (initial/medial/final/isolated).
+   However, it is strongly recommended to normalize any presentation form Arabic letters
+   to their non-presentation form before calling uroman.
+ * Added force flush directive ($|=1;).
+
+Changes in version 0.5 (minor upgrade)
+ * Improvements for Uyghur (make sure to use language option: -l uig)
+
+Changes in version 0.4 (minor upgrade)
+ * Improvements for Thai (special cases for vowel/consonant reordering, e.g. for "sara o"; dropped some aspiration 'h's)
+ * Minor change for Arabic (added "alef+fathatan" = "an")
+
+New features in version 0.3
+ * Covers Mandarin (Chinese)
+ * Improved romanization for numerous languages
+ * Preserves capitalization (e.g. from Latin, Cyrillic, Greek scripts)
+ * Maps from native digits to Western numbers
+ * Faster for South Asian languages
+
+### Other features
+ * Web interface: http://www.isi.edu/~ulf/uroman.html
+ * Vowelization is provided when locally computable, e.g. for many South Asian languages and Tibetan.
+
+### Limitations
+ * The current version of uroman has a few limitations, some of which we plan to address in future versions.
+   For Japanese, *uroman* currently romanizes hiragana and katakana as expected, but kanji are interpreted as Chinese characters and romanized as such. 
+   For Egyptian hieroglyphs, only single-sound phonetic characters and numbers are currently romanized. 
+   For Linear B, only phonetic syllabic characters are romanized. 
+   For some other extinct scripts such as cuneiform, no romanization is provided.
+ * A romanizer is not a full transliterator. For example, this version of
+   uroman does not vowelize text that lacks explicit vowelization such as
+   normal text in Arabic and Hebrew (without diacritics/points).
+
+### Acknowledgments
+This research is based upon work supported in part by the Office of the Director of National Intelligence (ODNI), Intelligence Advanced Research Projects Activity (IARPA), via contract # FA8650-17-C-9116, and by research sponsored by Air Force Research Laboratory (AFRL) under agreement number FA8750-19-1-1000. The views and conclusions contained herein are those of the authors and should not be interpreted as necessarily representing the official policies, either expressed or implied, of ODNI, IARPA, Air Force Laboratory, DARPA, or the U.S. Government. The U.S. Government is authorized to reproduce and distribute reprints for governmental purposes notwithstanding any copyright annotation therein.

uroman/README.txt

@@ -0,0 +1,141 @@
+uroman version 1.2.8
+Release date: April 23, 2021
+Author: Ulf Hermjakob, USC Information Sciences Institute
+
+uroman is a universal romanizer. It converts text in any script to the Latin alphabet.
+
+Usage: uroman.pl [-l <lang-code>] [--chart] [--no-cache] < STDIN
+       where the optional <lang-code> is a 3-letter languages code, e.g. ara, bel, bul, deu, ell, eng, fas,
+            grc, ell, eng, heb, kaz, kir, lav, lit, mkd, mkd2, oss, pnt, pus, rus, srp, srp2, tur, uig, ukr, yid.
+       --chart specifies chart output (in JSON format) to represent alternative romanizations.
+       --no-cache disables caching.
+Examples: bin/uroman.pl < text/zho.txt
+          bin/uroman.pl -l tur < text/tur.txt
+	  bin/uroman.pl -l heb --chart < text/heb.txt
+	  bin/uroman.pl < test/multi-script.txt > test/multi-script.uroman.txt
+
+Identifying the input as Arabic, Belarusian, Bulgarian, English, Farsi, German, 
+Ancient Greek, Modern Greek, Pontic Greek, Hebrew, Kazakh, Kyrgyz, Latvian, 
+Lithuanian, North Macedonian, Russian, Serbian, Turkish, Ukrainian, Uyghur or Yiddish 
+will improve romanization for those languages as some letters in those languages 
+have different sound values from other languages using the same script.
+No effect for other languages in this version.
+
+Bibliography: Ulf Hermjakob, Jonathan May, and Kevin Knight. 2018. Out-of-the-box universal romanization tool uroman. In Proceedings of the 56th Annual Meeting of Association for Computational Linguistics, Demo Track. [Best Demo Paper Award]
+
+Changes in version 1.2.8
+ * Improved support for Georgian.
+ * Updated UnicodeData.txt to version 13 (2021) with several new scripts (10% larger).
+ * Preserve various symbols (as opposed to mapping to the symbols' names).
+ * Various small improvements.
+Changes in version 1.2.7
+ * Improved support for Pashto.
+Changes in version 1.2.6
+ * Improved support for Ukrainian, Russian and Ogham (ancient Irish script).
+ * Added support for English Braille.
+ * Added alternative Romanization for North Macedonian and Serbian (mkd2/srp2)
+   reflecting a casual style that many native speakers of those languages use
+   when writing text in Latin script, e.g. non-accented single letters (e.g. "s")
+   rather than phonetically motivated combinations of letters (e.g. "sh").
+ * When a line starts with "::lcode xyz ", the new uroman version will switch to
+   that language for that line. This is used for the new reference test file.
+ * Various small improvements.
+Changes in version 1.2.5
+ * Improved support for Armenian and eight languages using Cyrillic scripts.
+   -- For Serbian and Macedonian, which are often written in both Cyrillic
+      and Latin scripts, uroman will map both official versions to the same
+      romanized text, e.g. both "Ниш" and "Niš" will be mapped to "Nish" (which
+      properly reflects the pronunciation of the city's name).
+      For both Serbian and Macedonian, casual writers often use a simplified
+      Latin form without diacritics, e.g. "s" to represent not only Cyrillic "с"
+      and Latin "s", but also "ш" or "š", even if this conflates "s" and "sh" and
+      other such pairs. The casual romanization can be simulated by using
+      alternative uroman language codes "srp2" and "mkd2", which romanize
+      both "Ниш" and "Niš" to "Nis" to reflect the casual Latin spelling.
+ * Various small improvements.
+Changes in version 1.2.4
+ * Added support for Tifinagh (a script used for Berber languages).
+ * Bug-fix that generated two emtpy lines for each empty line in cache mode.
+Changes in version 1.2.3
+ * Exclude emojis, dingbats, many other pictographs from being romanized (e.g. to "face")
+Changes in version 1.2
+ * Run-time improvement based on (1) token-based caching and (2) shortcut 
+   romanization (identity) of ASCII strings for default 1-best (non-chart) 
+   output. Speed-up by a factor of 10 for Bengali and Uyghur on medium and 
+   large size texts.
+ * Incremental improvements for Farsi, Amharic, Russian, Hebrew and related
+   languages.
+ * Richer lattice structure (more alternatives) for "Romanization" of English
+   to support better matching to romanizations of other languages.
+   Changes output only when --chart option is specified. No change in output for
+   default 1-best output, which for ASCII characters is always the input string.
+Changes in version 1.1 (major upgrade)
+ * Offers chart output (in JSON format) to represent alternative romanizations.
+   -- Location of first character is defined to be "line: 1, start:0, end:0".
+ * Incremental improvements of Hebrew and Greek romanization; Chinese numbers.
+ * Improved web-interface at http://www.isi.edu/~ulf/uroman.html
+   -- Shows corresponding original and romanization text in red
+      when hovering over a text segment.
+   -- Shows alternative romanizations when hovering over romanized text
+      marked by dotted underline.
+   -- Added right-to-left script detection and improved display for right-to-left
+      script text (as determined line by line).
+   -- On-page support for some scripts that are often not pre-installed on users'
+      computers (Burmese, Egyptian, Klingon).
+Changes in version 1.0 (major upgrade)
+ * Upgraded principal internal data structure from string to lattice.
+ * Improvements mostly in vowelization of South and Southeast Asian languages.
+ * Vocalic 'r' more consistently treated as vowel (no additional vowel added).
+ * Repetition signs (Japanese/Chinese/Thai/Khmer/Lao) are mapped to superscript 2.
+ * Japanese Katakana middle dots now mapped to ASCII space.
+ * Tibetan intersyllabic mark now mapped to middle dot (U+00B7).
+ * Some corrections regarding analysis of Chinese numbers.
+ * Many more foreign diacritics and punctuation marks dropped or mapped to ASCII.
+ * Zero-width characters dropped, except line/sentence-initial byte order marks.
+ * Spaces normalized to ASCII space.
+ * Fixed bug that in some cases mapped signs (such as dagger or bullet) to their verbal descriptions.
+ * Tested against previous version of uroman with a new uroman visual diff tool.
+ * Almost an order of magnitude faster.
+Changes in version 0.7 (minor upgrade)
+ * Added script uroman-quick.pl for Arabic script languages, incl. Uyghur.
+   Much faster, pre-caching mapping of Arabic to Latin characters, simple greedy processing.
+   Will not convert material from non-Arabic blocks such as any (somewhat unusual) Cyrillic
+   or Chinese characters in Uyghur texts.
+Changes in version 0.6 (minor upgrade)
+ * Added support for two letter characters used in Uzbek:
+   (1) character "ʻ" ("modifier letter turned comma", which modifies preceding "g" and "u" letters)
+   (2) character "ʼ" ("modifier letter apostrophe", which Uzbek uses to mark a glottal stop).
+   Both are now mapped to "'" (plain ASCII apostrophe).
+ * Added support for Uyghur vowel characters such as "ې" (Arabic e) and "ۆ" (Arabic oe)
+   even when they are not preceded by "ئ" (yeh with hamza above).
+ * Added support for Arabic semicolon "؛", Arabic ligature forms for phrases such as "ﷺ"
+   ("sallallahou alayhe wasallam" = "prayer of God be upon him and his family and peace")
+ * Added robustness for Arabic letter presentation forms (initial/medial/final/isolated).
+   However, it is strongly recommended to normalize any presentation form Arabic letters
+   to their non-presentation form before calling uroman.
+ * Added force flush directive ($|=1;).
+Changes in version 0.5 (minor upgrade)
+ * Improvements for Uyghur (make sure to use language option: -l uig)
+Changes in version 0.4 (minor upgrade)
+ * Improvements for Thai (special cases for vowel/consonant reordering, e.g. for "sara o"; dropped some aspiration 'h's)
+ * Minor change for Arabic (added "alef+fathatan" = "an")
+New features in version 0.3
+ * Covers Mandarin (Chinese)
+ * Improved romanization for numerous languages
+ * Preserves capitalization (e.g. from Latin, Cyrillic, Greek scripts)
+ * Maps from native digits to Western numbers
+ * Faster for South Asian languages
+
+Other features
+ * Web interface: http://www.isi.edu/~ulf/uroman.html
+ * Vowelization is provided when locally computable, e.g. for many South Asian
+   languages and Tibetan.
+
+Limitations
+ * This version of uroman assumes all CJK ideographs to be Mandarin (Chinese).
+   This means that Japanese kanji are incorrectly romanized; however, Japanese
+   hiragana and katakana are properly romanized.
+ * A romanizer is not a full transliterator. For example, this version of
+   uroman does not vowelize text that lacks explicit vowelization such as
+   normal text in Arabic and Hebrew (without diacritics/points).
+

uroman/bin/de-accent.pl

@@ -0,0 +1,201 @@
+#!/usr/bin/perl -w
+
+sub print_version {
+   print STDERR "$0 version 1.1\n";
+   print STDERR "   Author: Ulf Hermjakob\n";
+   print STDERR "   Last changed: March 14, 2011\n";
+}
+
+sub print_usage {
+   print STDERR "$0 [options] < with_accents.txt > without_accents.txt\n";
+   print STDERR "   -h or -help\n";
+   print STDERR "   -v or -version\n";
+}
+
+sub de_accent_string {
+   local($s) = @_;
+
+   # $s =~ tr/A-Z/a-z/;
+   unless (0) {
+      # Latin-1
+      if ($s =~ /\xC3[\x80-\xBF]/) {
+         $s =~ s/(À|Á|Â|Ã|Ä|Å)/A/g;
+         $s =~ s/Æ/Ae/g;
+         $s =~ s/Ç/C/g;
+         $s =~ s/Ð/D/g;
+         $s =~ s/(È|É|Ê|Ë)/E/g;
+         $s =~ s/(Ì|Í|Î|Ï)/I/g;
+         $s =~ s/Ñ/N/g;
+         $s =~ s/(Ò|Ó|Ô|Õ|Ö|Ø)/O/g;
+         $s =~ s/(Ù|Ú|Û|Ü)/U/g;
+         $s =~ s/Þ/Th/g;
+         $s =~ s/Ý/Y/g;
+         $s =~ s/(à|á|â|ã|ä|å)/a/g;
+         $s =~ s/æ/ae/g;
+         $s =~ s/ç/c/g;
+         $s =~ s/(è|é|ê|ë)/e/g;
+         $s =~ s/(ì|í|î|ï)/i/g;
+         $s =~ s/ð/d/g;
+         $s =~ s/ñ/n/g;
+         $s =~ s/(ò|ó|ô|õ|ö)/o/g;
+         $s =~ s/ß/ss/g;
+         $s =~ s/þ/th/g;
+         $s =~ s/(ù|ú|û|ü)/u/g;
+         $s =~ s/(ý|ÿ)/y/g;
+      }
+      # Latin Extended-A
+      if ($s =~ /[\xC4-\xC5][\x80-\xBF]/) {
+         $s =~ s/(Ā|Ă|Ą)/A/g;
+         $s =~ s/(ā|ă|ą)/a/g;
+         $s =~ s/(Ć|Ĉ|Ċ|Č)/C/g;
+         $s =~ s/(ć|ĉ|ċ|č)/c/g;
+         $s =~ s/(Ď|Đ)/D/g;
+         $s =~ s/(ď|đ)/d/g;
+         $s =~ s/(Ē|Ĕ|Ė|Ę|Ě)/E/g;
+         $s =~ s/(ē|ĕ|ė|ę|ě)/e/g;
+         $s =~ s/(Ĝ|Ğ|Ġ|Ģ)/G/g;
+         $s =~ s/(ĝ|ğ|ġ|ģ)/g/g;
+         $s =~ s/(Ĥ|Ħ)/H/g;
+         $s =~ s/(ĥ|ħ)/h/g;
+         $s =~ s/(Ĩ|Ī|Ĭ|Į|İ)/I/g;
+         $s =~ s/(ĩ|ī|ĭ|į|ı)/i/g;
+         $s =~ s/IJ/Ij/g;
+         $s =~ s/ij/ij/g;
+         $s =~ s/Ĵ/J/g;
+         $s =~ s/ĵ/j/g;
+         $s =~ s/Ķ/K/g;
+         $s =~ s/(ķ|ĸ)/k/g;
+         $s =~ s/(Ĺ|Ļ|Ľ|Ŀ|Ł)/L/g;
+         $s =~ s/(ļ|ľ|ŀ|ł)/l/g;
+         $s =~ s/(Ń|Ņ|Ň|Ŋ)/N/g;
+         $s =~ s/(ń|ņ|ň|ʼn|ŋ)/n/g;
+         $s =~ s/(Ō|Ŏ|Ő)/O/g;
+         $s =~ s/(ō|ŏ|ő)/o/g;
+         $s =~ s/Œ/Oe/g;
+         $s =~ s/œ/oe/g;
+         $s =~ s/(Ŕ|Ŗ|Ř)/R/g;
+         $s =~ s/(ŕ|ŗ|ř)/r/g;
+         $s =~ s/(Ś|Ŝ|Ş|Š)/S/g;
+         $s =~ s/(ś|ŝ|ş|š|ſ)/s/g;
+         $s =~ s/(Ţ|Ť|Ŧ)/T/g;
+         $s =~ s/(ţ|ť|ŧ)/t/g;
+         $s =~ s/(Ũ|Ū|Ŭ|Ů|Ű|Ų)/U/g;
+         $s =~ s/(ũ|ū|ŭ|ů|ű|ų)/u/g;
+         $s =~ s/Ŵ/W/g;
+         $s =~ s/ŵ/w/g;
+         $s =~ s/(Ŷ|Ÿ)/Y/g;
+         $s =~ s/ŷ/y/g;
+         $s =~ s/(Ź|Ż|Ž)/Z/g;
+         $s =~ s/(ź|ż|ž)/z/g;
+      }
+      # Latin Extended Additional
+      if ($s =~ /\xE1[\xB8-\xBF][\x80-\xBF]/) {
+          $s =~ s/(ḁ|ạ|ả|ấ|ầ|ẩ|ẫ|ậ|ắ|ằ|ẳ|ẵ|ặ|ẚ)/a/g;
+          $s =~ s/(ḃ|ḅ|ḇ)/b/g;
+          $s =~ s/(ḉ)/c/g;
+          $s =~ s/(ḋ|ḍ|ḏ|ḑ|ḓ)/d/g;
+          $s =~ s/(ḕ|ḗ|ḙ|ḛ|ḝ|ẹ|ẻ|ẽ|ế|ề|ể|ễ|ệ)/e/g;
+          $s =~ s/(ḟ)/f/g;
+          $s =~ s/(ḡ)/g/g;
+          $s =~ s/(ḣ|ḥ|ḧ|ḩ|ḫ)/h/g;
+          $s =~ s/(ḭ|ḯ|ỉ|ị)/i/g;
+          $s =~ s/(ḱ|ḳ|ḵ)/k/g;
+          $s =~ s/(ḷ|ḹ|ḻ|ḽ)/l/g;
+          $s =~ s/(ḿ|ṁ|ṃ)/m/g;
+          $s =~ s/(ṅ|ṇ|ṉ|ṋ)/m/g;
+          $s =~ s/(ọ|ỏ|ố|ồ|ổ|ỗ|ộ|ớ|ờ|ở|ỡ|ợ|ṍ|ṏ|ṑ|ṓ)/o/g;
+          $s =~ s/(ṕ|ṗ)/p/g;
+          $s =~ s/(ṙ|ṛ|ṝ|ṟ)/r/g;
+          $s =~ s/(ṡ|ṣ|ṥ|ṧ|ṩ|ẛ)/s/g;
+          $s =~ s/(ṫ|ṭ|ṯ|ṱ)/t/g;
+          $s =~ s/(ṳ|ṵ|ṷ|ṹ|ṻ|ụ|ủ|ứ|ừ|ử|ữ|ự)/u/g;
+          $s =~ s/(ṽ|ṿ)/v/g;
+          $s =~ s/(ẁ|ẃ|ẅ|ẇ|ẉ|ẘ)/w/g;
+          $s =~ s/(ẋ|ẍ)/x/g;
+          $s =~ s/(ẏ|ỳ|ỵ|ỷ|ỹ|ẙ)/y/g;
+          $s =~ s/(ẑ|ẓ|ẕ)/z/g;
+          $s =~ s/(Ḁ|Ạ|Ả|Ấ|Ầ|Ẩ|Ẫ|Ậ|Ắ|Ằ|Ẳ|Ẵ|Ặ)/A/g;
+          $s =~ s/(Ḃ|Ḅ|Ḇ)/B/g;
+          $s =~ s/(Ḉ)/C/g;
+          $s =~ s/(Ḋ|Ḍ|Ḏ|Ḑ|Ḓ)/D/g;
+          $s =~ s/(Ḕ|Ḗ|Ḙ|Ḛ|Ḝ|Ẹ|Ẻ|Ẽ|Ế|Ề|Ể|Ễ|Ệ)/E/g;
+          $s =~ s/(Ḟ)/F/g;
+          $s =~ s/(Ḡ)/G/g;
+          $s =~ s/(Ḣ|Ḥ|Ḧ|Ḩ|Ḫ)/H/g;
+          $s =~ s/(Ḭ|Ḯ|Ỉ|Ị)/I/g;
+          $s =~ s/(Ḱ|Ḳ|Ḵ)/K/g;
+          $s =~ s/(Ḷ|Ḹ|Ḻ|Ḽ)/L/g;
+          $s =~ s/(Ḿ|Ṁ|Ṃ)/M/g;
+          $s =~ s/(Ṅ|Ṇ|Ṉ|Ṋ)/N/g;
+          $s =~ s/(Ṍ|Ṏ|Ṑ|Ṓ|Ọ|Ỏ|Ố|Ồ|Ổ|Ỗ|Ộ|Ớ|Ờ|Ở|Ỡ|Ợ)/O/g;
+          $s =~ s/(Ṕ|Ṗ)/P/g;
+          $s =~ s/(Ṙ|Ṛ|Ṝ|Ṟ)/R/g;
+          $s =~ s/(Ṡ|Ṣ|Ṥ|Ṧ|Ṩ)/S/g;
+          $s =~ s/(Ṫ|Ṭ|Ṯ|Ṱ)/T/g;
+          $s =~ s/(Ṳ|Ṵ|Ṷ|Ṹ|Ṻ|Ụ|Ủ|Ứ|Ừ|Ử|Ữ|Ự)/U/g;
+          $s =~ s/(Ṽ|Ṿ)/V/g;
+          $s =~ s/(Ẁ|Ẃ|Ẅ|Ẇ|Ẉ)/W/g;
+          $s =~ s/(Ẍ)/X/g;
+          $s =~ s/(Ẏ|Ỳ|Ỵ|Ỷ|Ỹ)/Y/g; 
+          $s =~ s/(Ẑ|Ẓ|Ẕ)/Z/g;
+      }
+      # Greek letters
+      if ($s =~ /\xCE[\x86-\xAB]/) {
+          $s =~ s/ά/α/g;
+          $s =~ s/έ/ε/g;
+          $s =~ s/ί/ι/g;
+          $s =~ s/ϊ/ι/g;
+          $s =~ s/ΐ/ι/g;
+          $s =~ s/ό/ο/g;
+          $s =~ s/ύ/υ/g;
+          $s =~ s/ϋ/υ/g;
+          $s =~ s/ΰ/υ/g;
+          $s =~ s/ώ/ω/g;
+          $s =~ s/Ά/Α/g;
+          $s =~ s/Έ/Ε/g;
+          $s =~ s/Ή/Η/g;
+          $s =~ s/Ί/Ι/g;
+          $s =~ s/Ϊ/Ι/g;
+          $s =~ s/Ύ/Υ/g;
+          $s =~ s/Ϋ/Υ/g;
+          $s =~ s/Ώ/Ω/g;
+      }
+      # Cyrillic letters
+      if ($s =~ /\xD0[\x80-\xAF]/) {
+          $s =~ s/Ѐ/Е/g;
+          $s =~ s/Ё/Е/g;
+          $s =~ s/Ѓ/Г/g;
+          $s =~ s/Ќ/К/g;
+          $s =~ s/Ѝ/И/g;
+          $s =~ s/Й/И/g;
+          $s =~ s/ѐ/е/g;
+          $s =~ s/ё/е/g;
+          $s =~ s/ѓ/г/g;
+          $s =~ s/ќ/к/g;
+          $s =~ s/ѝ/и/g;
+          $s =~ s/й/и/g;
+      }
+   }
+   return $s;
+}
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-*(h|help)$/i) {
+      &print_usage;
+      exit 1;
+   } elsif ($arg =~ /^-*(v|version)$/i) {
+      &print_version;
+      exit 1;
+   } else {
+      print STDERR "Ignoring unrecognized argument $arg\n";
+   }
+}
+
+$line_number = 0;
+while (<>) {
+   $line_number++;
+   print &de_accent_string($_);
+}
+exit 0;
+

uroman/bin/string-distance.pl

@@ -0,0 +1,99 @@
+#!/usr/bin/perl -w
+
+# Author: Ulf Hermjakob
+# Release date: October 13, 2019
+
+# Usage: string-distance.pl {-lc1 <language-code>} {-lc2 <language-code>} < STDIN > STDOUT
+# Example: string-distance.pl -lc1 rus -lc2 ukr < STDIN > STDOUT
+# Example: string-distance.pl < ../test/string-similarity-test-input.txt
+# Input format: two strings per line (tab-separated, in Latin script)
+#    Strings in non-Latin scripts should first be romanized. (Recommended script: uroman.pl)
+# Output format: repetition of the two input strings, plus the string distance between them (tab-separated).
+#    Additional output meta info lines at the top are marked with an initial #.
+#
+# The script uses data from a string-distance-cost-rules file that lists costs,
+# where the default cost is "1" with lower costs for differences in vowels,
+# duplicate consonants, "f" vs. "ph" etc.
+# Language cost rules can be language-specific and context-sensitive.
+
+$|=1;
+
+use FindBin;
+use Cwd "abs_path";
+use File::Basename qw(dirname);
+use File::Spec;
+
+my $bin_dir = abs_path(dirname($0));
+my $root_dir = File::Spec->catfile($bin_dir, File::Spec->updir());
+my $data_dir = File::Spec->catfile($root_dir, "data");
+my $lib_dir = File::Spec->catfile($root_dir, "lib");
+
+use lib "$FindBin::Bin/../lib";
+use List::Util qw(min max);
+use NLP::utilities;
+use NLP::stringDistance;
+$util = NLP::utilities;
+$sd = NLP::stringDistance;
+$verbose = 0;
+$separator = "\t";
+
+$cost_rule_filename = File::Spec->catfile($data_dir, "string-distance-cost-rules.txt");
+
+$lang_code1 = "eng";
+$lang_code2 = "eng";
+%ht = ();
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-+lc1$/) {
+      $lang_code_candidate = shift @ARGV;
+      $lang_code1 = $lang_code_candidate if $lang_code_candidate =~ /^[a-z]{3,3}$/;
+   } elsif ($arg =~ /^-+lc2$/) {
+      $lang_code_candidate = shift @ARGV;
+      $lang_code2 = $lang_code_candidate if $lang_code_candidate =~ /^[a-z]{3,3}$/;
+   } elsif ($arg =~ /^-+(v|verbose)$/) {
+      $verbose = shift @ARGV;
+   } else {
+      print STDERR "Ignoring unrecognized arg $arg\n";
+   }
+}
+
+$sd->load_string_distance_data($cost_rule_filename, *ht, $verbose);
+print STDERR "Loaded resources.\n" if $verbose;
+
+my $chart_id = 0;
+my $line_number = 0;
+print "# Lang-code-1: $lang_code1 Lang-code-2: $lang_code2\n";
+while (<>) {
+   $line_number++;
+   if ($verbose) {
+      if ($line_number =~ /000$/) {
+         if ($line_number =~ /0000$/) {
+	    print STDERR $line_number;
+         } else {
+	    print STDERR ".";
+         }
+      }
+   }
+   my $line = $_;
+   $line =~ s/^\xEF\xBB\xBF//;
+   next if $line =~ /^\s*(\#.*)?$/;
+   my $s1;
+   my $s2;
+   if (($s1, $s2) = ($line =~ /^("(?:\\"|[^"])*"|\S+)$separator("(?:\\"|[^"])*"|\S+)\s*$/)) {
+      $s1 = $util->dequote_string($s1);
+      $s2 = $util->dequote_string($s2);
+   } elsif ($line =~ /^\s*(#.*)$/) {
+   } else {
+      print STDERR "Could not process line $line_number: $line" if $verbose;
+      print "\n";
+      next;
+   }
+
+   $cost = $sd->quick_romanized_string_distance_by_chart($s1, $s2, *ht, "", $lang_code1, $lang_code2);
+   print "$s1\t$s2\t$cost\n";
+}
+print STDERR "\n" if $verbose;
+
+exit 0;
+

uroman/bin/uroman-quick.pl

@@ -0,0 +1,58 @@
+#!/usr/bin/perl -w
+
+# uroman  Nov. 12, 2015 - July 25, 2016
+# version v0.7
+# Author: Ulf Hermjakob
+
+# Usage: uroman-quick.pl {-l [tur|uig|ukr|yid]} < STDIN
+# currently only for Arabic script languages, incl. Uyghur
+
+$|=1;
+
+use FindBin;
+use Cwd "abs_path";
+use File::Basename qw(dirname);
+use File::Spec;
+
+my $bin_dir = abs_path(dirname($0));
+my $root_dir = File::Spec->catfile($bin_dir, File::Spec->updir());
+my $data_dir = File::Spec->catfile($root_dir, "data");
+my $lib_dir = File::Spec->catfile($root_dir, "lib");
+
+use lib "$FindBin::Bin/../lib";
+use NLP::Romanizer;
+use NLP::UTF8;
+$romanizer = NLP::Romanizer;
+%ht = ();
+$lang_code = "";
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-+(l|lc|lang-code)$/) {
+      $lang_code = lc (shift @ARGV || "")
+   } else {
+      print STDERR "Ignoring unrecognized arg $arg\n";
+   }
+}
+
+$romanization_table_arabic_block_filename = File::Spec->catfile($data_dir, "romanization-table-arabic-block.txt");
+$romanization_table_filename = File::Spec->catfile($data_dir, "romanization-table.txt");
+
+$romanizer->load_romanization_table(*ht, $romanization_table_arabic_block_filename);
+$romanizer->load_romanization_table(*ht, $romanization_table_filename);
+
+$line_number = 0;
+while (<>) {
+   $line_number++;
+   my $line = $_;
+   print $romanizer->quick_romanize($line, $lang_code, *ht) . "\n";
+   if ($line_number =~ /0000$/) {
+      print STDERR $line_number;
+   } elsif ($line_number =~ /000$/) {
+      print STDERR ".";
+   }
+}
+print STDERR "\n";
+
+exit 0;
+

uroman/bin/uroman-tsv.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Created by Thamme Gowda on June 17, 2019
+
+DIR=$(dirname "${BASH_SOURCE[0]}")  # get the directory name
+# DIR=$(realpath "${DIR}")    # resolve its full path if need be
+
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+    >&2 echo "ERROR: invalid args"
+    >&2 echo "Usage: <input.tsv> [<output.tsv>]"
+    exit 2
+fi
+
+INP=$1
+OUT=$2
+
+CMD=$DIR/uroman.pl
+
+function romanize(){
+    paste <(cut -f1 $INP) <(cut -f2 $INP | $CMD)
+}
+
+if [[ -n $OUT ]]; then
+    romanize > $OUT
+else
+    romanize
+fi
+
+

uroman/bin/uroman.pl

@@ -0,0 +1,138 @@
+#!/usr/bin/perl -w
+
+# uroman  Nov. 12, 2015 - Apr. 23, 2021
+$version = "v1.2.8";
+# Author: Ulf Hermjakob
+
+# Usage: uroman.pl {-l [ara|bel|bul|deu|ell|eng|fas|grc|heb|kaz|kir|lav|lit|mkd|mkd2|oss|pnt|rus|srp|srp2|tur|uig|ukr|yid]} {--chart|--offset-mapping} {--no-cache} {--workset} < STDIN
+# Example: cat workset.txt | uroman.pl --offset-mapping --workset
+
+$|=1;
+
+use FindBin;
+use Cwd "abs_path";
+use File::Basename qw(dirname);
+use File::Spec;
+
+my $bin_dir = abs_path(dirname($0));
+my $root_dir = File::Spec->catfile($bin_dir, File::Spec->updir());
+my $data_dir = File::Spec->catfile($root_dir, "data");
+my $lib_dir = File::Spec->catfile($root_dir, "lib");
+
+use lib "$FindBin::Bin/../lib";
+use NLP::Chinese;
+use NLP::Romanizer;
+use NLP::UTF8;
+use NLP::utilities;
+use JSON;
+$chinesePM = NLP::Chinese;
+$romanizer = NLP::Romanizer;
+$util = NLP::utilities;
+%ht = ();
+%pinyin_ht = ();
+$lang_code = "";
+$return_chart_p = 0;
+$return_offset_mappings_p = 0;
+$workset_p = 0;
+$cache_rom_tokens_p = 1;
+
+$script_data_filename = File::Spec->catfile($data_dir, "Scripts.txt");
+$unicode_data_overwrite_filename = File::Spec->catfile($data_dir, "UnicodeDataOverwrite.txt");
+$unicode_data_filename = File::Spec->catfile($data_dir, "UnicodeData.txt");
+$romanization_table_filename = File::Spec->catfile($data_dir, "romanization-table.txt");
+$chinese_tonal_pinyin_filename = File::Spec->catfile($data_dir, "Chinese_to_Pinyin.txt");
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-+(l|lc|lang-code)$/) {
+      $lang_code = lc (shift @ARGV || "")
+   } elsif ($arg =~ /^-+chart$/i) {
+      $return_chart_p = 1;
+   } elsif ($arg =~ /^-+workset$/i) {
+      $workset_p = 1;
+   } elsif ($arg =~ /^-+offset[-_]*map/i) {
+      $return_offset_mappings_p = 1;
+   } elsif ($arg =~ /^-+unicode[-_]?data/i) {
+      $filename = shift @ARGV;
+      if (-r $filename) {
+         $unicode_data_filename = $filename;
+      } else {
+         print STDERR "Ignoring invalid UnicodeData filename $filename\n";
+      }
+   } elsif ($arg =~ /^-+(no-tok-cach|no-cach)/i) {
+      $cache_rom_tokens_p = 0;
+   } else {
+      print STDERR "Ignoring unrecognized arg $arg\n";
+   }
+}
+
+$romanizer->load_script_data(*ht, $script_data_filename);
+$romanizer->load_unicode_data(*ht, $unicode_data_filename);
+$romanizer->load_unicode_overwrite_romanization(*ht, $unicode_data_overwrite_filename);
+$romanizer->load_romanization_table(*ht, $romanization_table_filename);
+$chinese_to_pinyin_not_yet_loaded_p = 1;
+$current_date = $util->datetime("dateTtime");
+$lang_code_clause = ($lang_code) ? " \"lang-code\":\"$lang_code\",\n" : "";
+
+print "{\n \"romanizer\":\"uroman $version (Ulf Hermjakob, USC/ISI)\",\n \"date\":\"$current_date\",\n$lang_code_clause \"romanization\": [\n" if $return_chart_p;
+my $line_number = 0;
+my $chart_result = "";
+while (<>) {
+   $line_number++;
+   my $line = $_;
+   my $snt_id = "";
+   if ($workset_p) {
+      next if $line =~ /^#/;
+      if (($i_value, $s_value) = ($line =~ /^(\S+\.\d+)\s(.*)$/)) {
+	 $snt_id = $i_value;
+	 $line = "$s_value\n";
+      } else {
+	 next;
+      }
+   }
+   if ($chinese_to_pinyin_not_yet_loaded_p && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($line)) {
+      $chinesePM->read_chinese_tonal_pinyin_files(*pinyin_ht, $chinese_tonal_pinyin_filename);
+      $chinese_to_pinyin_not_yet_loaded_p = 0;
+   }
+   if ($return_chart_p) {
+      print $chart_result;
+      *chart_ht = $romanizer->romanize($line, $lang_code, "", *ht, *pinyin_ht, 0, "return chart", $line_number);
+      $chart_result = $romanizer->chart_to_json_romanization_elements(0, $chart_ht{N_CHARS}, *chart_ht, $line_number);
+   } elsif ($return_offset_mappings_p) {
+      ($best_romanization, $offset_mappings) = $romanizer->romanize($line, $lang_code, "", *ht, *pinyin_ht, 0, "return offset mappings", $line_number, 0);
+      print "::snt-id $snt_id\n" if $workset_p;
+      print "::orig $line";
+      print "::rom $best_romanization\n";
+      print "::align $offset_mappings\n\n";
+   } elsif ($cache_rom_tokens_p) {
+      print $romanizer->romanize_by_token_with_caching($line, $lang_code, "", *ht, *pinyin_ht, 0, "", $line_number) . "\n";
+   } else {
+      print $romanizer->romanize($line, $lang_code, "", *ht, *pinyin_ht, 0, "", $line_number) . "\n";
+   }
+}
+$chart_result =~ s/,(\s*)$/$1/;
+print $chart_result;
+print " ]\n}\n" if $return_chart_p;
+
+$dev_test_p = 0;
+if ($dev_test_p) {
+   $n_suspicious_code_points = 0;
+   $n_instances = 0;
+   foreach $char_name (sort { hex($ht{UTF_NAME_TO_UNICODE}->{$a}) <=> hex($ht{UTF_NAME_TO_UNICODE}->{$b}) }
+                            keys %{$ht{SUSPICIOUS_ROMANIZATION}}) {
+      $unicode_value = $ht{UTF_NAME_TO_UNICODE}->{$char_name};
+      $utf8_string = $ht{UTF_NAME_TO_CODE}->{$char_name};
+      foreach $romanization (sort keys %{$ht{SUSPICIOUS_ROMANIZATION}->{$char_name}}) {
+         $count = $ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization};
+	 $s = ($count == 1) ? "" : "s";
+         print STDERR "*** Suspiciously lengthy romanization:\n" unless $n_suspicious_code_points;
+         print STDERR "::s $utf8_string ::t $romanization ::comment $char_name (U+$unicode_value)\n";
+	 $n_suspicious_code_points++;
+	 $n_instances += $count;
+      }
+   }
+   print STDERR "  *** Total of $n_suspicious_code_points suspicious code points ($n_instances instance$s)\n" if $n_suspicious_code_points;
+} 
+
+exit 0;
+

uroman/data/Scripts.txt

@@ -0,0 +1,135 @@
+::script-name Aegean
+::script-name Ahom
+::script-name Anatolian Hieroglyph
+::script-name Arabic ::direction right-to-left
+::script-name Armenian
+::script-name Avestan
+::script-name Balinese
+::script-name Bamum
+::script-name Bassa Vah
+::script-name Batak
+::script-name Bengali ::abugida-default-vowel a
+::script-name Bhaiksuki
+::script-name Bopomofo ::language Chinese
+::script-name Brahmi ::abugida-default-vowel a
+::script-name Braille
+::script-name Buginese
+::script-name Buhid
+::script-name Canadian Syllabics
+::script-name Carian
+::script-name Caucasian Albanian
+::script-name Chakma
+::script-name Cham
+::script-name Cherokee
+::script-name Coptic
+::script-name Cuneiform
+::script-name Cypriot
+::script-name Cyrillic
+::script-name CJK ::alt-script-name Chinese, Kanji ::language Chinese, Japanese, Korean, Mandarin
+::script-name Deseret
+::script-name Devanagari ::abugida-default-vowel a
+::script-name Duployan
+::script-name Egyptian Hieroglyph
+::script-name Elbasan
+::script-name Ethiopic
+::script-name Georgian
+::script-name Glagolitic
+::script-name Gothic
+::script-name Grantha
+::script-name Greek
+::script-name Gujarati ::abugida-default-vowel a
+::script-name Gurmukhi ::abugida-default-vowel a
+::script-name Hangul ::language Korean
+::script-name Hanunoo
+::script-name Hatran
+::script-name Hebrew ::direction right-to-left
+::script-name Hiragana ::language Japanese
+::script-name Imperial Aramaic
+::script-name Inscriptional Pahlavi
+::script-name Inscriptional Parthian
+::script-name Javanese
+::script-name Kaithi
+::script-name Kannada ::abugida-default-vowel a
+::script-name Katakana ::language Japanese
+::script-name Kayah Li
+::script-name Kharoshthi
+::script-name Khmer ::abugida-default-vowel a, o
+::script-name Khojki
+::script-name Khudawadi
+::script-name Klingon
+::script-name Lao
+::script-name Lepcha
+::script-name Latin
+::script-name Limbu
+::script-name Linear A
+::script-name Linear B
+::script-name Lycian
+::script-name Lydian
+::script-name Mahajani
+::script-name Malayalam ::abugida-default-vowel a
+::script-name Mandaic
+::script-name Manichaean
+::script-name Marchen
+::script-name Meetei Mayek
+::script-name Meroitic Cursive
+::script-name Meroitic Hieroglyphic
+::script-name Miao
+::script-name Modi ::abugida-default-vowel a
+::script-name Mongolian
+::script-name Mro
+::script-name Multani
+::script-name Myanmar ::alt-script-name Burmese ::abugida-default-vowel a
+::script-name Nabataean
+::script-name New Tai Lue
+::script-name Newa
+::script-name Nko ::direction right-to-left
+::script-name Ogham
+::script-name Ol Chiki
+::script-name Old Hungarian
+::script-name Old Italic
+::script-name Old Permic
+::script-name Old Persian
+::script-name Old North Arabian
+::script-name Old South Arabian
+::script-name Old Turkic
+::script-name Oriya ::alt-script-name Odia ::abugida-default-vowel a
+::script-name Osage
+::script-name Osmanya
+::script-name Pahawh Hmong
+::script-name Palmyrene
+::script-name Pau Cin Hau
+::script-name Phags-pa
+::script-name Phaistos Disc
+::script-name Phoenician
+::script-name Psalter Pahlavi
+::script-name Rejang
+::script-name Runic
+::script-name Samaritan
+::script-name Saurashtra
+::script-name Sharada
+::script-name Shavian
+::script-name Siddham
+::script-name Sinhala ::abugida-default-vowel a
+::script-name Sora Sompeng
+::script-name Sundanese ::abugida-default-vowel a
+::script-name Syloti Nagri
+::script-name Syriac
+::script-name Tagalog
+::script-name Tagbanwa
+::script-name Tai Le
+::script-name Tai Tham
+::script-name Tai Viet
+::script-name Takri
+::script-name Tamil ::abugida-default-vowel a
+::script-name Tangut
+::script-name Telugu ::abugida-default-vowel a
+::script-name Thaana ::direction right-to-left
+::script-name Thai
+::script-name Tibetan ::abugida-default-vowel a
+::script-name Tifinagh
+::script-name Tirhuta
+::script-name Ugaritic
+::script-name Vai
+::script-name Vedic
+::script-name Warang Citi
+::script-name Yi

uroman/data/UnicodeDataOverwrite.txt

@@ -0,0 +1,442 @@
+## UnicodeDataOverwrite.txt
+::u 00A0 ::r " "  ::comment no-break space
+::u 01BF ::r w  ::comment ƿ Latin Character Wynn (Old English)
+::u 0294 ::r ' ::comment gottal stop
+::u 0295 ::r ' ::comment ʕ voiced pharyngeal fricative
+::u 0305 ::r ""  ::comment ̅ Combining overline
+::u 0306 ::r ""  ::comment ̆ Combining breve
+::u 0307 ::r ""  ::comment ̇ Combining dot above
+::u 030A ::r ""  ::comment ̊ Combining ring above
+::u 030C ::r ""  ::comment ̌ Combining caron
+::u 0311 ::r ""  ::comment ̑ Combining inverted breve
+::u 031D ::r ""  ::comment ̝ Combining down up below
+::u 031E ::r ""  ::comment ̞ Combining down tack below
+::u 031F ::r ""  ::comment ̟ Combining plus sign below
+::u 0323 ::r ""  ::comment ̣ Combining dot below
+::u 0325 ::r ""  ::comment ̥ Combining ring below
+::u 0329 ::r ""  ::comment ̩ Combining vertical line below
+::u 032A ::r ""  ::comment ̪ Combining bridge below
+::u 032F ::r ""  ::comment ̯ Combining inverted breve below
+::u 0342 ::r ""  ::comment ͂ Combining Greek perispomeni (circumflex accent)
+::u 0343 ::r ""  ::comment ̓ Combining Greek koronis
+::u 0361 ::r ""  ::comment Combining double inverted breve
+::u 0384 ::r ""  ::comment ΄ Greek tonos
+::u 0482 ::r 1000·  ::comment ҂ Cyrillic thousands sign
+::u 0483 ::r ""  ::comment ҃ Combining Cyrillic Titlo ::annotation titlo 
+::u 0484 ::r ""  ::comment ҄ Combining Cyrillic Palatalization ::annotation palatalization 
+::u 055B ::r ""  ::comment ՛ Armenian emphasis mark
+::u 055F ::r ""  ::comment ՟ Armenian abbreviation mark ::annotation abbreviation
+
+::u 0901 ::r +m  ::comment Devanagari sign candrabindu
+::u 0902 ::r +m  ::comment Devanagari sign anusvara
+::u 0903 ::r +h  ::comment Devanagari sign visarga
+::u 093D ::r '   ::comment Devanagari sign avagraha
+::u 0950 ::r om  ::comment ॐ Devanagari om symbol
+::u 0951 ::r ""  ::comment ॑ Devanagari stress sign "udatta"
+::u 0952 ::r ""  ::comment ॒ Devanagari stress sign "anudatta" 
+::u 0981 ::r +n  ::comment Bengali sign candrabindu ("chôndrôbindu")
+::u 0982 ::r +ng ::comment Bengali sign anusvara ("ônushar")
+::u 0983 ::r +h  ::comment Bengali sign visarga ("bishôrgô")
+::u 099A ::r ch  ::comment instead of Bengali C(A)
+::u 099B ::r chh ::comment instead of Bengali CC(A)
+::u 0A02 ::r +m  ::comment Gurmukhi sign bindi
+::u 0A70 ::r +m  ::comment Gurmukhi tippi
+# ::u 0A72 ::r "" ::comment Gurmukhi addak
+::u 0A72 ::r ""  ::comment Gurmukhi iri
+::u 0A73 ::r ""  ::comment Gurmukhi ura
+::u 0B01 ::r +m  ::comment Oriya sign candrabindu
+::u 0B03 ::r +h  ::comment Oriya sign visarga
+::u 0B5F ::r ya  ::comment ୟ Oriya letter yya
+::u 0B82 ::r +m  ::comment Tamil sign anusvara (not to be used?)
+::u 0B83 ::r +h  ::comment Tamil sign visarga ("āytam")
+::u 0B9F ::r t   ::comment instead of Tamil TT(A)
+::u 0BA3 ::r n   ::comment instead of Tamil NN(A)
+::u 0BA9 ::r n   ::comment instead of Tamil NNN(A)
+::u 0BB1 ::r r   ::comment instead of Tamil RR(A)
+::u 0BB3 ::r l   ::comment instead of Tamil LL(A)
+::u 0BB4 ::r l   ::comment instead of Tamil LLL(A)
+::u 0C03 ::r +h  ::comment ః Telugu sign visarga
+::u 0C83 ::r +h  ::comment Kannada sign visarga
+::u 0D02 ::r +m  ::comment Malayalam sign anusvara
+::u 0D03 ::r +h  ::comment Malayalam sign visarga
+::u 0D82 ::r +n  ::comment Sinhala sign anusvaraya
+::u 0DA4 ::r ny ::comment Sinhala ඤ
+::u 0DA5 ::r gn ::comment Sinhala ඥ
+::u 0DCA ::r ""  ::comment Sinhala sign al-lakuna (virama = no vowel)
+::u 0DCF ::r aa ::comment Sinhala ා
+::u 0DD0 ::r ae ::comment Sinhala ැ
+::u 0DD1 ::r ae ::comment Sinhala ෑ
+::u 0DD2 ::r i ::comment Sinhala ි
+::u 0DD3 ::r ii ::comment Sinhala ී
+::u 0DD4 ::r u ::comment Sinhala ු
+::u 0DD6 ::r uu ::comment Sinhala ූ
+::u 0DD8 ::r r ::comment Sinhala ෘ
+::u 0DD9 ::r e ::comment Sinhala ෙ
+::u 0DDA ::r ee ::comment Sinhala ේ
+::u 0DDB ::r ai ::comment Sinhala ෛ
+::u 0DDC ::r o ::comment Sinhala ො
+::u 0DDD ::r oo ::comment Sinhala ෝ
+::u 0DDE ::r au ::comment Sinhala ෞ
+::u 0DDF ::r aa ::comment Sinhala ා
+::u 0DF2 ::r rr ::comment Sinhala ෲ
+
+::u 0E02 ::r k   ::comment Thai character KHO KHAI
+::u 0E03 ::r k   ::comment Thai character KHO KHUAT
+::u 0E04 ::r k   ::comment Thai character KHO KHWAI
+::u 0E05 ::r k   ::comment Thai character KHO KHON
+::u 0E06 ::r k   ::comment Thai character KHO RAKHANG
+::u 0E10 ::r t   ::comment Thai character THO THAN
+::u 0E11 ::r t   ::comment Thai character THO NANGMONTHO
+::u 0E12 ::r t   ::comment Thai character THO PHUTHAO
+::u 0E16 ::r t   ::comment Thai character THO THUNG
+::u 0E17 ::r t   ::comment Thai character THO THAHAN
+::u 0E18 ::r t   ::comment Thai character THO THONG
+::u 0E1C ::r p   ::comment Thai character PHO PHUNG
+::u 0E1E ::r p   ::comment Thai character PHO PHAN
+::u 0E20 ::r p   ::comment Thai character PHO SAMPHAO
+::u 0E2D ::r o   ::comment Thai character O ANG
+::u 0E2F ::r ...  ::comment ฯ Thai character PAIYANNOI (ellipsis, abbreviation)
+::u 0E31 ::r a   ::comment Thai character MAI HAN-AKAT
+::u 0E3A ::r ""  ::comment Thai character PHINTHU (Pali virama)
+::u 0E40 ::r e   ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA E
+::u 0E41 ::r ae  ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA AE
+::u 0E42 ::r o   ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA O
+::u 0E43 ::r ai  ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA AI MAIMUAN
+::u 0E44 ::r ai  ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA AI MAIMALAI
+::u 0E45 ::r ""  ::comment Thai character LAKKHANGYAO vowel lengthener
+::u 0E47 ::r o   ::comment Thai character MAITAIKHU vowel shortener
+::u 0E48 ::r ""  ::tone-mark non-standard ::comment Thai tone mark MAI EK
+::u 0E49 ::r ""  ::tone-mark standard ::comment Thai tone mark MAI THO
+::u 0E4A ::r ""  ::tone-mark high ::comment Thai tone mark MAI TRI
+::u 0E4B ::r ""  ::tone-mark rising ::comment Thai tone mark MAI CHATTAWA
+::u 0E4C ::r ""  ::comment Thai character THANTHAKHAT cancellation mark (cf. virama)
+::u 0E4D ::r +m  ::comment ํ Thai character NIKHAHIT final nasal (cf. anusvara)
+::u 0ECC ::r ""  ::comment ໌ Lao cancellation mark ::annotation cancellation
+::u 0F0B ::r ·  ::comment ་ Tibetan mark intersyllabic tsheg
+::u 0F0C ::r ""  ::comment ༌ Tibetan mark delimiter tsheg bstar
+::u 0F84 ::r ""  ::comment ྄ Tibetan halanta
+::u 1036 ::r +n  ::comment Myanmar sign anusvara ("auk myit")
+::u 1037 ::r ""  ::tone-mark creaky ::comment Myanmar sign dot below
+::u 1038 ::r ""  ::tone-mark high ::comment Myanmar sign visarga
+
+::u 16A0 ::r f ::comment ᚠ RUNIC LETTER FEHU FEOH FE F
+::u 16A1 ::r v ::comment ᚡ RUNIC LETTER V
+::u 16A2 ::r u ::comment ᚢ RUNIC LETTER URUZ UR U
+::u 16A3 ::r y ::comment ᚣ RUNIC LETTER YR
+::u 16A4 ::r y ::comment ᚤ RUNIC LETTER Y
+::u 16A5 ::r w ::comment ᚥ RUNIC LETTER W
+::u 16A6 ::r th ::comment ᚦ RUNIC LETTER THURISAZ THURS THORN
+::u 16A7 ::r th ::comment ᚧ RUNIC LETTER ETH
+::u 16A8 ::r a ::comment ᚨ RUNIC LETTER ANSUZ A
+::u 16A9 ::r o ::comment ᚩ RUNIC LETTER OS O
+::u 16AA ::r a ::comment ᚪ RUNIC LETTER AC A
+::u 16AB ::r ae ::comment ᚫ RUNIC LETTER AESC
+::u 16AC ::r o ::comment ᚬ RUNIC LETTER LONG-BRANCH-OSS O
+::u 16AD ::r o ::comment ᚭ RUNIC LETTER SHORT-TWIG-OSS O
+::u 16AE ::r o ::comment ᚮ RUNIC LETTER O
+::u 16AF ::r oe ::comment ᚯ RUNIC LETTER OE
+::u 16B0 ::r on ::comment ᚰ RUNIC LETTER ON
+::u 16B1 ::r r ::comment ᚱ RUNIC LETTER RAIDO RAD REID R
+::u 16B2 ::r k ::comment ᚲ RUNIC LETTER KAUNA
+::u 16B3 ::r c ::comment ᚳ RUNIC LETTER CEN
+::u 16B4 ::r k ::comment ᚴ RUNIC LETTER KAUN K
+::u 16B5 ::r g ::comment ᚵ RUNIC LETTER G
+::u 16B6 ::r ng ::comment ᚶ RUNIC LETTER ENG
+::u 16B7 ::r g ::comment ᚷ RUNIC LETTER GEBO GYFU G
+::u 16B8 ::r g ::comment ᚸ RUNIC LETTER GAR
+::u 16B9 ::r w ::comment ᚹ RUNIC LETTER WUNJO WYNN W
+::u 16BA ::r h ::comment ᚺ RUNIC LETTER HAGLAZ H
+::u 16BB ::r h ::comment ᚻ RUNIC LETTER HAEGL H
+::u 16BC ::r h ::comment ᚼ RUNIC LETTER LONG-BRANCH-HAGALL H
+::u 16BD ::r h ::comment ᚽ RUNIC LETTER SHORT-TWIG-HAGALL H
+::u 16BE ::r n ::comment ᚾ RUNIC LETTER NAUDIZ NYD NAUD N
+::u 16BF ::r n ::comment ᚿ RUNIC LETTER SHORT-TWIG-NAUD N
+::u 16C0 ::r n ::comment ᛀ RUNIC LETTER DOTTED-N
+::u 16C1 ::r i ::comment ᛁ RUNIC LETTER ISAZ IS ISS I
+::u 16C2 ::r e ::comment ᛂ RUNIC LETTER E
+::u 16C3 ::r j ::comment ᛃ RUNIC LETTER JERAN J
+::u 16C4 ::r j ::comment ᛄ RUNIC LETTER GER
+::u 16C5 ::r ae ::comment ᛅ RUNIC LETTER LONG-BRANCH-AR AE
+::u 16C6 ::r a ::comment ᛆ RUNIC LETTER SHORT-TWIG-AR A
+::u 16C7 ::r i ::comment ᛇ RUNIC LETTER IWAZ EOH
+::u 16C8 ::r p ::comment ᛈ RUNIC LETTER PERTHO PEORTH P
+::u 16C9 ::r z ::comment ᛉ RUNIC LETTER ALGIZ EOLHX
+::u 16CA ::r s ::comment ᛊ RUNIC LETTER SOWILO S
+::u 16CB ::r s ::comment ᛋ RUNIC LETTER SIGEL LONG-BRANCH-SOL S
+::u 16CC ::r s ::comment ᛌ RUNIC LETTER SHORT-TWIG-SOL S
+::u 16CD ::r c ::comment ᛍ RUNIC LETTER C
+::u 16CE ::r z ::comment ᛎ RUNIC LETTER Z
+::u 16CF ::r t ::comment ᛏ RUNIC LETTER TIWAZ TIR TYR T
+::u 16D0 ::r t ::comment ᛐ RUNIC LETTER SHORT-TWIG-TYR T
+::u 16D1 ::r d ::comment ᛑ RUNIC LETTER D
+::u 16D2 ::r b ::comment ᛒ RUNIC LETTER BERKANAN BEORC BJARKAN B
+::u 16D3 ::r b ::comment ᛓ RUNIC LETTER SHORT-TWIG-BJARKAN B
+::u 16D4 ::r p ::comment ᛔ RUNIC LETTER DOTTED-P
+::u 16D5 ::r p ::comment ᛕ RUNIC LETTER OPEN-P
+::u 16D6 ::r e ::comment ᛖ RUNIC LETTER EHWAZ EH E
+::u 16D7 ::r m ::comment ᛗ RUNIC LETTER MANNAZ MAN M
+::u 16D8 ::r m ::comment ᛘ RUNIC LETTER LONG-BRANCH-MADR M
+::u 16D9 ::r m ::comment ᛙ RUNIC LETTER SHORT-TWIG-MADR M
+::u 16DA ::r l ::comment ᛚ RUNIC LETTER LAUKAZ LAGU LOGR L
+::u 16DB ::r l ::comment ᛛ RUNIC LETTER DOTTED-L
+::u 16DC ::r ng ::comment ᛜ RUNIC LETTER INGWAZ
+::u 16DD ::r ng ::comment ᛝ RUNIC LETTER ING
+::u 16DE ::r d ::comment ᛞ RUNIC LETTER DAGAZ DAEG D
+::u 16DF ::r o ::comment ᛟ RUNIC LETTER OTHALAN ETHEL O
+::u 16E0 ::r ea ::comment ᛠ RUNIC LETTER EAR
+::u 16E1 ::r io ::comment ᛡ RUNIC LETTER IOR
+::u 16E2 ::r q ::comment ᛢ RUNIC LETTER CWEORTH
+::u 16E3 ::r k ::comment ᛣ RUNIC LETTER CALC
+::u 16E4 ::r k ::comment ᛤ RUNIC LETTER CEALC
+::u 16E5 ::r st ::comment ᛥ RUNIC LETTER STAN
+::u 16E6 ::r r ::comment ᛦ RUNIC LETTER LONG-BRANCH-YR
+::u 16E7 ::r r ::comment ᛧ RUNIC LETTER SHORT-TWIG-YR
+::u 16E8 ::r r ::comment ᛨ RUNIC LETTER ICELANDIC-YR
+::u 16E9 ::r q ::comment ᛩ RUNIC LETTER Q
+::u 16EA ::r x ::comment ᛪ RUNIC LETTER X
+
+::u 17B9 ::r oe ::comment Khmer vowel sign y (short)
+::u 17BA ::r oe ::comment Khmer vowel sign yy (long)
+::u 17C6 ::r +m  ::comment Khmer sign nikahit (cf. anusvara)
+::u 17C7 ::r +h  ::comment Khmer sign reahmuk (cf. visarga)
+::u 17C8 ::r '   ::comment Khmer sign yuukaleapintu (short vowel and glottal stop)
+::u 17C9 ::r ""  ::comment Khmer sign muusikatoan: changes the second register to the first
+::u 17CA ::r ""  ::comment Khmer sign triisap: changes the first register to the second
+::u 17CB ::r ""  ::comment Khmer sign bantoc (vowel shortener)
+::u 17D2 ::r ""  ::comment Khmer sign coeng (foot/subscript, cf. virama = no vowel)
+::u 17D5 ::r .  ::comment Khmer sign bariyoosan; period ending entire text or chapter
+
+::u 180E ::r '  ::comment ᠎ Mongolian vowel separator
+
+::u 1B80 ::r +ng  ::comment ᮀ Sundanese sign panyecek
+::u 1B81 ::r +r  ::comment ᮁ Sundanese sign panglayar
+::u 1B82 ::r +h  ::comment ᮂ Sundanese sign pangwisad
+::u 1BA1 ::r ya  ::comment ᮡ Sundanese consonant sign pamingkal
+::u 1BA2 ::r ra  ::comment ᮢ Sundanese consonant sign panyakr
+::u 1BA3 ::r la  ::comment ᮣ Sundanese consonant sign panyiku
+::u 1BA4 ::r i  ::comment ᮤ Sundanese consonant sign panghulu
+::u 1BA5 ::r u  ::comment ᮥ Sundanese consonant sign panyuku
+::u 1BA6 ::r e  ::comment ᮦ Sundanese vowel sign panaelaeng
+::u 1BA7 ::r o  ::comment ᮧ Sundanese vowel sign panolong
+::u 1BA8 ::r e  ::comment ᮨ Sundanese vowel sign pamepet
+::u 1BA9 ::r eu  ::comment ᮩ Sundanese vowel sign paneuleung
+::u 1BAA ::r ""  ::comment ᮪ Sundanese sign pamaaeh or patén (no vowel/virama)
+
+::u 1FBD ::r ""  ::comment ᾽ Greek koronis 
+::u 1FFE ::r ""  ::comment Greek dasia (rough breathing)
+
+::u 2002 ::r " "  ::comment en space
+::u 2003 ::r " "  ::comment em space
+::u 2004 ::r " "  ::comment three-per-em space
+::u 2005 ::r " "  ::comment four-per-em space
+::u 2006 ::r " "  ::comment six-per-em space
+::u 2007 ::r " "  ::comment figure space
+::u 2008 ::r " "  ::comment punctuation space
+::u 2009 ::r " "  ::comment thin space
+::u 200A ::r " "  ::comment hair space
+::u 202F ::r " "  ::comment narrow no-break space
+
+::u 2D30 ::r a ::comment TIFINAGH LETTER YA ⴰ
+::u 2D31 ::r b ::comment TIFINAGH LETTER YAB ⴱ
+::u 2D32 ::r bh ::comment TIFINAGH LETTER YABH ⴲ
+::u 2D33 ::r g ::comment TIFINAGH LETTER YAG ⴳ
+::u 2D34 ::r ghh ::comment TIFINAGH LETTER YAGHH ⴴ
+::u 2D35 ::r j ::comment TIFINAGH LETTER BERBER ACADEMY YAJ ⴵ
+::u 2D36 ::r j ::comment TIFINAGH LETTER YAJ ⴶ
+::u 2D37 ::r d ::comment TIFINAGH LETTER YAD ⴷ
+::u 2D38 ::r dh ::comment TIFINAGH LETTER YADH ⴸ
+::u 2D39 ::r dd ::comment TIFINAGH LETTER YADD ⴹ
+::u 2D3A ::r ddh ::comment TIFINAGH LETTER YADDH ⴺ
+::u 2D3B ::r e ::comment TIFINAGH LETTER YEY ⴻ
+::u 2D3C ::r f ::comment TIFINAGH LETTER YAF ⴼ
+::u 2D3D ::r k ::comment TIFINAGH LETTER YAK ⴽ
+::u 2D3E ::r k ::comment TIFINAGH LETTER TUAREG YAK ⴾ
+::u 2D3F ::r khh ::comment TIFINAGH LETTER YAKHH ⴿ
+::u 2D40 ::r h ::comment TIFINAGH LETTER YAH ⵀ
+::u 2D41 ::r h ::comment TIFINAGH LETTER BERBER ACADEMY YAH ⵁ
+::u 2D42 ::r h ::comment TIFINAGH LETTER TUAREG YAH ⵂ
+::u 2D43 ::r hh ::comment TIFINAGH LETTER YAHH ⵃ
+::u 2D44 ::r ' ::comment TIFINAGH LETTER YAA ⵄ
+::u 2D45 ::r kh ::comment TIFINAGH LETTER YAKH ⵅ
+::u 2D46 ::r kh ::comment TIFINAGH LETTER TUAREG YAKH ⵆ
+::u 2D47 ::r q ::comment TIFINAGH LETTER YAQ ⵇ
+::u 2D48 ::r q ::comment TIFINAGH LETTER TUAREG YAQ ⵈ
+::u 2D49 ::r i ::comment TIFINAGH LETTER YI ⵉ
+::u 2D4A ::r zh ::comment TIFINAGH LETTER YAZH ⵊ
+::u 2D4B ::r zh ::comment TIFINAGH LETTER AHAGGAR YAZH ⵋ
+::u 2D4C ::r zh ::comment TIFINAGH LETTER TUAREG YAZH ⵌ
+::u 2D4D ::r l ::comment TIFINAGH LETTER YAL ⵍ
+::u 2D4E ::r m ::comment TIFINAGH LETTER YAM ⵎ
+::u 2D4F ::r n ::comment TIFINAGH LETTER YAN ⵏ
+::u 2D50 ::r gn ::comment TIFINAGH LETTER TUAREG YAGN ⵐ
+::u 2D51 ::r ng ::comment TIFINAGH LETTER TUAREG YANG ⵑ
+::u 2D52 ::r p ::comment TIFINAGH LETTER YAP ⵒ
+::u 2D53 ::r u ::comment TIFINAGH LETTER YU ⵓ
+::u 2D54 ::r r ::comment TIFINAGH LETTER YAR ⵔ
+::u 2D55 ::r rr ::comment TIFINAGH LETTER YARR ⵕ
+::u 2D56 ::r gh ::comment TIFINAGH LETTER YAGH ⵖ
+::u 2D57 ::r gh ::comment TIFINAGH LETTER TUAREG YAGH ⵗ
+::u 2D58 ::r gh ::comment TIFINAGH LETTER AYER YAGH ⵘ
+::u 2D59 ::r s ::comment TIFINAGH LETTER YAS ⵙ
+::u 2D5A ::r ss ::comment TIFINAGH LETTER YASS ⵚ
+::u 2D5B ::r sh ::comment TIFINAGH LETTER YASH ⵛ
+::u 2D5C ::r t ::comment TIFINAGH LETTER YAT ⵜ
+::u 2D5D ::r th ::comment TIFINAGH LETTER YATH ⵝ
+::u 2D5E ::r ch ::comment TIFINAGH LETTER YACH ⵞ
+::u 2D5F ::r tt ::comment TIFINAGH LETTER YATT ⵟ
+::u 2D60 ::r v ::comment TIFINAGH LETTER YAV ⵠ
+::u 2D61 ::r w ::comment TIFINAGH LETTER YAW ⵡ
+::u 2D62 ::r y ::comment TIFINAGH LETTER YAY ⵢ
+::u 2D63 ::r z ::comment TIFINAGH LETTER YAZ ⵣ
+::u 2D64 ::r z ::comment TIFINAGH LETTER TAWELLEMET YAZ ⵤ
+::u 2D65 ::r zz ::comment TIFINAGH LETTER YAZZ ⵥ
+::u 2D66 ::r ye ::comment TIFINAGH LETTER YE ⵦ
+::u 2D67 ::r yo ::comment TIFINAGH LETTER YO ⵧ
+::u 2D6F ::r "" ::comment TIFINAGH MODIFIER LETTER LABIALIZATION MARK ⵯ
+::u 2D70 ::r "" ::comment TIFINAGH SEPARATOR MARK ⵰
+::u 2D7F ::r "" ::comment TIFINAGH CONSONANT JOINER ⵿
+
+::u 3063 ::r tsu ::comment Hiragana letter small tsu
+::u 30C3 ::r tsu ::comment Katakana letter small tsu
+
+::u ABE3 ::r o  ::comment ꯣ Meetei Mayek vowel sign onap
+::u ABE7 ::r ou  ::comment ꯧ Meetei Mayek vowel sign sounap
+
+::u F008 ::r ""  ::comment Yoruba diacritic in private use area
+::u F00F ::r ""  ::comment Yoruba diacritic in private use area
+::u F023 ::r ""  ::comment Yoruba diacritic in private use area
+::u F025 ::r ""  ::comment Yoruba diacritic in private use area
+
+::u F8D0 ::r a   ::name KLINGON LETTER A
+::u F8D1 ::r b   ::name KLINGON LETTER B
+::u F8D2 ::r ch  ::name KLINGON LETTER CH
+::u F8D3 ::r D   ::name KLINGON LETTER D
+::u F8D4 ::r e   ::name KLINGON LETTER E
+::u F8D5 ::r gh  ::name KLINGON LETTER GH
+::u F8D6 ::r H   ::name KLINGON LETTER H
+::u F8D7 ::r I   ::name KLINGON LETTER I
+::u F8D8 ::r j   ::name KLINGON LETTER J
+::u F8D9 ::r l   ::name KLINGON LETTER L
+::u F8DA ::r m   ::name KLINGON LETTER M
+::u F8DB ::r n   ::name KLINGON LETTER N
+::u F8DC ::r ng  ::name KLINGON LETTER NG
+::u F8DD ::r o   ::name KLINGON LETTER O
+::u F8DE ::r p   ::name KLINGON LETTER P
+::u F8DF ::r q   ::name KLINGON LETTER Q
+::u F8E0 ::r Q   ::name KLINGON LETTER Q
+::u F8E1 ::r r   ::name KLINGON LETTER R
+::u F8E2 ::r S   ::name KLINGON LETTER S
+::u F8E3 ::r t   ::name KLINGON LETTER T
+::u F8E4 ::r tlh ::name KLINGON LETTER TLH
+::u F8E5 ::r u   ::name KLINGON LETTER U
+::u F8E6 ::r v   ::name KLINGON LETTER V
+::u F8E7 ::r w   ::name KLINGON LETTER W
+::u F8E8 ::r y   ::name KLINGON LETTER Y
+::u F8E9 ::r '   ::name KLINGON LETTER GLOTTAL STOP
+::u F8F0 ::num 0 ::name KLINGON DIGIT ZERO
+::u F8F1 ::num 1 ::name KLINGON DIGIT ONE
+::u F8F2 ::num 2 ::name KLINGON DIGIT TWO
+::u F8F3 ::num 3 ::name KLINGON DIGIT THREE
+::u F8F4 ::num 4 ::name KLINGON DIGIT FOUR
+::u F8F5 ::num 5 ::name KLINGON DIGIT FIVE
+::u F8F6 ::num 6 ::name KLINGON DIGIT SIX
+::u F8F7 ::num 7 ::name KLINGON DIGIT SEVEN
+::u F8F8 ::num 8 ::name KLINGON DIGIT EIGHT
+::u F8F9 ::num 9 ::name KLINGON DIGIT NINE
+::u F8FD ::r ,   ::name KLINGON COMMA
+::u F8FE ::r .   ::name KLINGON FULL STOP
+::u F8FF ::name KLINGON MUMMIFICATION GLYPH
+
+::u 1163D ::r +m ::comment Modi sign anusvara
+::u 1163E ::r +h ::comment Modi sign visarga
+
+::u 13068 ::num 1000000 ::comment Egyptian Hieroglyph
+::u 1308B ::r r ::comment Egyptian Hieroglyph ::pic mouth
+::u 1309D ::r ' ::comment Egyptian Hieroglyph (ayn) ::pic forearm
+::u 130A7 ::r d ::comment Egyptian Hieroglyph ::pic hand
+::u 130AD ::num 10000 ::comment Egyptian Hieroglyph
+::u 130AE ::num 20000 ::comment Egyptian Hieroglyph
+::u 130AF ::num 30000 ::comment Egyptian Hieroglyph
+::u 130B0 ::num 40000 ::comment Egyptian Hieroglyph
+::u 130B1 ::num 50000 ::comment Egyptian Hieroglyph
+::u 130B2 ::num 60000 ::comment Egyptian Hieroglyph
+::u 130B3 ::num 70000 ::comment Egyptian Hieroglyph
+::u 130B4 ::num 80000 ::comment Egyptian Hieroglyph
+::u 130B5 ::num 90000 ::comment Egyptian Hieroglyph
+::u 130B6 ::num 50000 ::comment Egyptian Hieroglyph
+::u 130C0 ::r b ::comment Egyptian Hieroglyph ::pic foot
+::u 130ED ::r l ::comment Egyptian Hieroglyph [also rw] ::pic lion recumbent 
+::u 13121 ::r h ::comment Egyptian Hieroglyph (f-underscore) ::pic aninal's belly and udder 
+::u 1313F ::r a ::comment Egyptian Hieroglyph (alef) ::pic vulture
+::u 13153 ::r m ::comment Egyptian Hieroglyph ::pic owl
+::u 13171 ::r w ::comment Egyptian Hieroglyph ::pic quail chick
+::u 13187 ::r ::comment Egyptian Hieroglyph (determinative/son) H8 ::pic egg
+::u 13190 ::num 100000 ::comment Egyptian Hieroglyph
+::u 13191 ::r f ::comment Egyptian Hieroglyph ::pic horned viper
+::u 13193 ::r d ::comment Egyptian Hieroglyph (J) ::pic cobra
+::u 131BC ::num 1000 ::comment Egyptian Hieroglyph
+::u 131BD ::num 2000 ::comment Egyptian Hieroglyph
+::u 131BE ::num 3000 ::comment Egyptian Hieroglyph
+::u 131BF ::num 4000 ::comment Egyptian Hieroglyph
+::u 131C0 ::num 5000 ::comment Egyptian Hieroglyph
+::u 131C1 ::num 6000 ::comment Egyptian Hieroglyph
+::u 131C2 ::num 7000 ::comment Egyptian Hieroglyph
+::u 131C3 ::num 8000 ::comment Egyptian Hieroglyph
+::u 131C4 ::num 9000 ::comment Egyptian Hieroglyph
+::u 131CB ::r i ::comment Egyptian Hieroglyph (yod) ::pic single reed 
+::u 131CC ::r y ::comment Egyptian Hieroglyph ::pic double reed 
+::u 1320E ::r q ::comment Egyptian Hieroglyph (qaf) ::pic sandy slope
+::u 13209 ::comment Egyptian Hieroglyph ::pic desert hills
+::u 13216 ::r n ::comment Egyptian Hieroglyph ::pic ripple of water
+::u 13219 ::r sh ::comment Egyptian Hieroglyph (š) ::pic basin 
+::u 13254 ::r h ::comment Egyptian Hieroglyph ::pic reed shelter
+::u 13283 ::r z ::comment Egyptian Hieroglyph [also S?] ::pic door bolt 
+::u 132AA ::r p ::comment Egyptian Hieroglyph ::pic stool
+::u 132D4 ::r n ::comment Egyptian Hieroglyph ::pic red crown
+::u 132F4 ::r s ::comment Egyptian Hieroglyph [also Z?] ::pic folded cloth 
+::u 13319 ::comment Egyptian Hieroglyph ::pic throw stick
+::u 13362 ::num 100 ::comment Egyptian Hieroglyph
+::u 13363 ::num 200 ::comment Egyptian Hieroglyph
+::u 13364 ::num 300 ::comment Egyptian Hieroglyph
+::u 13365 ::num 400 ::comment Egyptian Hieroglyph
+::u 13366 ::num 500 ::comment Egyptian Hieroglyph
+::u 13367 ::num 600 ::comment Egyptian Hieroglyph
+::u 13368 ::num 700 ::comment Egyptian Hieroglyph
+::u 13369 ::num 800 ::comment Egyptian Hieroglyph
+::u 1336A ::num 900 ::comment Egyptian Hieroglyph
+::u 1336B ::num 500 ::comment Egyptian Hieroglyph
+::u 1336F ::r o ::comment Egyptian Hieroglyph ::pic lasso
+::u 1337F ::r t ::comment Egyptian Hieroglyph (ṯ) ::pic hobble 
+::u 13386 ::num 10 ::comment Egyptian Hieroglyph
+::u 13387 ::num 20 ::comment Egyptian Hieroglyph
+::u 13388 ::num 30 ::comment Egyptian Hieroglyph
+::u 13389 ::num 40 ::comment Egyptian Hieroglyph
+::u 1338A ::num 50 ::comment Egyptian Hieroglyph
+::u 1338B ::num 60 ::comment Egyptian Hieroglyph
+::u 1338C ::num 70 ::comment Egyptian Hieroglyph
+::u 1338D ::num 80 ::comment Egyptian Hieroglyph
+::u 1338E ::num 90 ::comment Egyptian Hieroglyph
+::u 1338F ::num 20 ::comment Egyptian Hieroglyph
+::u 13390 ::num 30 ::comment Egyptian Hieroglyph
+::u 13391 ::num 40 ::comment Egyptian Hieroglyph
+::u 13392 ::num 50 ::comment Egyptian Hieroglyph
+::u 1339B ::r h ::comment Egyptian Hieroglyph ::pic twisted flax
+::u 133A1 ::r k ::comment Egyptian Hieroglyph ::pic basket with handle
+::u 133A2 ::r k ::comment Egyptian Hieroglyph ::pic basket with handle, variant
+::u 133A4 ::r g ::comment Egyptian Hieroglyph ::pic bag
+::u 133BC ::r g ::comment Egyptian Hieroglyph ::pic stand
+::u 133CF ::r t ::comment Egyptian Hieroglyph ::pic loaf
+::u 133ED ::r y ::comment Egyptian Hieroglyph ::pic two strokes
+::u 133F2 ::r w ::comment Egyptian Hieroglyph ::pic quail chick, hieratic variant         
+::u 133FA ::num 1 ::comment Egyptian Hieroglyph
+::u 133FB ::num 2 ::comment Egyptian Hieroglyph
+::u 133FC ::num 3 ::comment Egyptian Hieroglyph
+::u 133FD ::num 4 ::comment Egyptian Hieroglyph
+::u 133FE ::num 5 ::comment Egyptian Hieroglyph
+::u 133FF ::num 6 ::comment Egyptian Hieroglyph
+::u 13400 ::num 7 ::comment Egyptian Hieroglyph
+::u 13401 ::num 8 ::comment Egyptian Hieroglyph
+::u 13402 ::num 9 ::comment Egyptian Hieroglyph
+::u 13403 ::num 5 ::comment Egyptian Hieroglyph
+::u 1340D ::r kh ::comment Egyptian Hieroglyph (ḫ, khah) ::pic placenta?
+::u 1341D ::r m ::comment Egyptian Hieroglyph (also jm)

uroman/data/romanization-table-arabic-block.txt

@@ -0,0 +1,179 @@
+::s ، ::t , ::comment ARABIC COMMA
+::s ؛ ::t ; ::comment ARABIC SEMICOLON
+::s ؟ ::t ? ::comment ARABIC QUESTION MARK
+::s ء ::t ' ::comment ARABIC LETTER HAMZA
+::s آ ::t a ::comment ARABIC LETTER ALEF WITH MADDA ABOVE
+::s أ ::t a ::comment ARABIC LETTER ALEF WITH HAMZA ABOVE
+::s ؤ ::t w ::comment ARABIC LETTER WAW WITH HAMZA ABOVE
+::s إ ::t i ::comment ARABIC LETTER ALEF WITH HAMZA BELOW
+::s ئ ::t ye ::comment ARABIC LETTER YEH WITH HAMZA ABOVE
+::s ا ::t a ::comment ARABIC LETTER ALEF
+::s ب ::t b ::comment ARABIC LETTER BEH
+::s ة ::t a ::comment ARABIC LETTER TEH MARBUTA
+::s ت ::t t ::comment ARABIC LETTER TEH
+::s ث ::t th ::comment ARABIC LETTER THEH
+::s ج ::t j ::comment ARABIC LETTER JEEM
+::s ح ::t h ::comment ARABIC LETTER HAH
+::s خ ::t kh ::comment ARABIC LETTER KHAH
+::s د ::t d ::comment ARABIC LETTER DAL
+::s ذ ::t th ::comment ARABIC LETTER THAL
+::s ر ::t r ::comment ARABIC LETTER REH
+::s ز ::t z ::comment ARABIC LETTER ZAIN
+::s س ::t s ::comment ARABIC LETTER SEEN
+::s ش ::t sh ::comment ARABIC LETTER SHEEN
+::s ص ::t s ::comment ARABIC LETTER SAD
+::s ض ::t d ::comment ARABIC LETTER DAD
+::s ط ::t t ::comment ARABIC LETTER TAH
+::s ظ ::t z ::comment ARABIC LETTER ZAH
+::s ع ::t ' ::comment ARABIC LETTER AIN
+::s غ ::t gh ::comment ARABIC LETTER GHAIN
+::s ـ ::t - ::comment ARABIC TATWEEL
+::s ف ::t f ::comment ARABIC LETTER FEH
+::s ق ::t q ::comment ARABIC LETTER QAF
+::s ك ::t k ::comment ARABIC LETTER KAF
+::s ل ::t l ::comment ARABIC LETTER LAM
+::s م ::t m ::comment ARABIC LETTER MEEM
+::s ن ::t n ::comment ARABIC LETTER NOON
+::s ه ::t h ::comment ARABIC LETTER HEH
+::s و ::t w ::comment ARABIC LETTER WAW
+::s ى ::t a ::comment ARABIC LETTER ALEF MAKSURA
+::s ي ::t y ::comment ARABIC LETTER YEH
+::s َ ::t a ::comment ARABIC FATHA
+::s ُ ::t u ::comment ARABIC DAMMA
+::s ِ ::t i ::comment ARABIC KASRA
+::s ْ ::t  ::comment ARABIC SUKUN
+::s ٔ ::t ' ::comment ARABIC HAMZA ABOVE
+::s ٕ ::t ' ::comment ARABIC HAMZA BELOW
+::s ٠ ::t 0 ::comment ARABIC-INDIC DIGIT ZERO
+::s ١ ::t 1 ::comment ARABIC-INDIC DIGIT ONE
+::s ٢ ::t 2 ::comment ARABIC-INDIC DIGIT TWO
+::s ٣ ::t 3 ::comment ARABIC-INDIC DIGIT THREE
+::s ٤ ::t 4 ::comment ARABIC-INDIC DIGIT FOUR
+::s ٥ ::t 5 ::comment ARABIC-INDIC DIGIT FIVE
+::s ٦ ::t 6 ::comment ARABIC-INDIC DIGIT SIX
+::s ٧ ::t 7 ::comment ARABIC-INDIC DIGIT SEVEN
+::s ٨ ::t 8 ::comment ARABIC-INDIC DIGIT EIGHT
+::s ٩ ::t 9 ::comment ARABIC-INDIC DIGIT NINE
+::s ٪ ::t % ::comment ARABIC PERCENT SIGN
+::s ٫ ::t , ::comment ARABIC DECIMAL SEPARATOR
+::s ٬ ::t , ::comment ARABIC THOUSANDS SEPARATOR
+::s ٮ ::t b ::comment ARABIC LETTER DOTLESS BEH
+::s ٯ ::t q ::comment ARABIC LETTER DOTLESS QAF
+::s ٰ ::t a ::comment ARABIC LETTER SUPERSCRIPT ALEF
+::s ٱ ::t a ::comment ARABIC LETTER ALEF WASLA
+::s ٲ ::t a ::comment ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE
+::s ٳ ::t a ::comment ARABIC LETTER ALEF WITH WAVY HAMZA BELOW
+::s ٷ ::t u ::comment ARABIC LETTER U WITH HAMZA ABOVE
+::s ٹ ::t tt ::comment ARABIC LETTER TTEH
+::s ٺ ::t tt ::comment ARABIC LETTER TTEHEH
+::s ٻ ::t b ::comment ARABIC LETTER BEEH
+::s ټ ::t t ::comment ARABIC LETTER TEH WITH RING
+::s ٽ ::t t ::comment ARABIC LETTER TEH WITH THREE DOTS ABOVE DOWNWARDS
+::s پ ::t p ::comment ARABIC LETTER PEH
+::s ٿ ::t t ::comment ARABIC LETTER TEHEH
+::s ڀ ::t b ::comment ARABIC LETTER BEHEH
+::s ځ ::t h ::comment ARABIC LETTER HAH WITH HAMZA ABOVE
+::s ڂ ::t h ::comment ARABIC LETTER HAH WITH TWO DOTS VERTICAL ABOVE
+::s ڃ ::t ny ::comment ARABIC LETTER NYEH
+::s ڄ ::t dy ::comment ARABIC LETTER DYEH
+::s څ ::t h ::comment ARABIC LETTER HAH WITH THREE DOTS ABOVE
+::s چ ::t tch ::comment ARABIC LETTER TCHEH
+::s ڇ ::t tch ::comment ARABIC LETTER TCHEHEH
+::s ڈ ::t dd ::comment ARABIC LETTER DDAL
+::s ډ ::t d ::comment ARABIC LETTER DAL WITH RING
+::s ڊ ::t d ::comment ARABIC LETTER DAL WITH DOT BELOW
+::s ڋ ::t d ::comment ARABIC LETTER DAL WITH DOT BELOW AND SMALL TAH
+::s ڌ ::t d ::comment ARABIC LETTER DAHAL
+::s ڍ ::t dd ::comment ARABIC LETTER DDAHAL
+::s ڎ ::t d ::comment ARABIC LETTER DUL
+::s ڏ ::t d ::comment ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS
+::s ڐ ::t d ::comment ARABIC LETTER DAL WITH FOUR DOTS ABOVE
+::s ڑ ::t rr ::comment ARABIC LETTER RREH
+::s ڒ ::t r ::comment ARABIC LETTER REH WITH SMALL V
+::s ړ ::t r ::comment ARABIC LETTER REH WITH RING
+::s ڔ ::t r ::comment ARABIC LETTER REH WITH DOT BELOW
+::s ڕ ::t r ::comment ARABIC LETTER REH WITH SMALL V BELOW
+::s ږ ::t r ::comment ARABIC LETTER REH WITH DOT BELOW AND DOT ABOVE
+::s ڗ ::t r ::comment ARABIC LETTER REH WITH TWO DOTS ABOVE
+::s ژ ::t j ::comment ARABIC LETTER JEH
+::s ڙ ::t r ::comment ARABIC LETTER REH WITH FOUR DOTS ABOVE
+::s ښ ::t s ::comment ARABIC LETTER SEEN WITH DOT BELOW AND DOT ABOVE
+::s ڛ ::t s ::comment ARABIC LETTER SEEN WITH THREE DOTS BELOW
+::s ڜ ::t s ::comment ARABIC LETTER SEEN WITH THREE DOTS BELOW AND THREE DOTS ABOVE
+::s ڝ ::t s ::comment ARABIC LETTER SAD WITH TWO DOTS BELOW
+::s ڞ ::t s ::comment ARABIC LETTER SAD WITH THREE DOTS ABOVE
+::s ڟ ::t t ::comment ARABIC LETTER TAH WITH THREE DOTS ABOVE
+::s ڠ ::t n ::comment ARABIC LETTER AIN WITH THREE DOTS ABOVE
+::s ڡ ::t f ::comment ARABIC LETTER DOTLESS FEH
+::s ڢ ::t f ::comment ARABIC LETTER FEH WITH DOT MOVED BELOW
+::s ڣ ::t f ::comment ARABIC LETTER FEH WITH DOT BELOW
+::s ڤ ::t v ::comment ARABIC LETTER VEH
+::s ڥ ::t f ::comment ARABIC LETTER FEH WITH THREE DOTS BELOW
+::s ڦ ::t p ::comment ARABIC LETTER PEHEH
+::s ڧ ::t q ::comment ARABIC LETTER QAF WITH DOT ABOVE
+::s ڨ ::t q ::comment ARABIC LETTER QAF WITH THREE DOTS ABOVE
+::s ک ::t k ::comment ARABIC LETTER KEHEH
+::s ڪ ::t k ::comment ARABIC LETTER SWASH KAF
+::s ګ ::t k ::comment ARABIC LETTER KAF WITH RING
+::s ڬ ::t k ::comment ARABIC LETTER KAF WITH DOT ABOVE
+::s ڭ ::t ng ::comment ARABIC LETTER NG
+::s ڮ ::t k ::comment ARABIC LETTER KAF WITH THREE DOTS BELOW
+::s گ ::t g ::comment ARABIC LETTER GAF
+::s ڰ ::t g ::comment ARABIC LETTER GAF WITH RING
+::s ڱ ::t ng ::comment ARABIC LETTER NGOEH
+::s ڲ ::t g ::comment ARABIC LETTER GAF WITH TWO DOTS BELOW
+::s ڳ ::t g ::comment ARABIC LETTER GUEH
+::s ڴ ::t g ::comment ARABIC LETTER GAF WITH THREE DOTS ABOVE
+::s ڵ ::t l ::comment ARABIC LETTER LAM WITH SMALL V
+::s ڶ ::t l ::comment ARABIC LETTER LAM WITH DOT ABOVE
+::s ڷ ::t l ::comment ARABIC LETTER LAM WITH THREE DOTS ABOVE
+::s ڸ ::t l ::comment ARABIC LETTER LAM WITH THREE DOTS BELOW
+::s ڹ ::t n ::comment ARABIC LETTER NOON WITH DOT BELOW
+::s ں ::t n ::comment ARABIC LETTER NOON GHUNNA
+::s ڻ ::t rn ::comment ARABIC LETTER RNOON
+::s ڼ ::t n ::comment ARABIC LETTER NOON WITH RING
+::s ڽ ::t n ::comment ARABIC LETTER NOON WITH THREE DOTS ABOVE
+::s ھ ::t h ::comment ARABIC LETTER HEH DOACHASHMEE
+::s ڿ ::t tch ::comment ARABIC LETTER TCHEH WITH DOT ABOVE
+::s ۀ ::t h ::comment ARABIC LETTER HEH WITH YEH ABOVE
+::s ہ ::t h ::comment ARABIC LETTER HEH GOAL
+::s ۂ ::t h ::comment ARABIC LETTER HEH GOAL WITH HAMZA ABOVE
+::s ۃ ::t a ::comment ARABIC LETTER TEH MARBUTA GOAL
+::s ۄ ::t w ::comment ARABIC LETTER WAW WITH RING
+::s ۅ ::t oe ::comment ARABIC LETTER KIRGHIZ OE
+::s ۆ ::t oe ::comment ARABIC LETTER OE
+::s ۇ ::t u ::comment ARABIC LETTER U
+::s ۈ ::t yu ::comment ARABIC LETTER YU
+::s ۉ ::t yu ::comment ARABIC LETTER KIRGHIZ YU
+::s ۊ ::t w ::comment ARABIC LETTER WAW WITH TWO DOTS ABOVE
+::s ۋ ::t v ::comment ARABIC LETTER VE
+::s ی ::t y ::comment ARABIC LETTER FARSI YEH
+::s ۍ ::t y ::comment ARABIC LETTER YEH WITH TAIL
+::s ێ ::t y ::comment ARABIC LETTER YEH WITH SMALL V
+::s ۏ ::t w ::comment ARABIC LETTER WAW WITH DOT ABOVE
+::s ې ::t e ::comment ARABIC LETTER E
+::s ۑ ::t y ::comment ARABIC LETTER YEH WITH THREE DOTS BELOW
+::s ے ::t y ::comment ARABIC LETTER YEH BARREE
+::s ۓ ::t y ::comment ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
+::s ۔ ::t . ::comment ARABIC FULL STOP
+::s ە ::t ae ::comment ARABIC LETTER AE
+::s ۮ ::t d ::comment ARABIC LETTER DAL WITH INVERTED V
+::s ۯ ::t r ::comment ARABIC LETTER REH WITH INVERTED V
+::s ۰ ::t 0 ::comment EXTENDED ARABIC-INDIC DIGIT ZERO
+::s ۱ ::t 1 ::comment EXTENDED ARABIC-INDIC DIGIT ONE
+::s ۲ ::t 2 ::comment EXTENDED ARABIC-INDIC DIGIT TWO
+::s ۳ ::t 3 ::comment EXTENDED ARABIC-INDIC DIGIT THREE
+::s ۴ ::t 4 ::comment EXTENDED ARABIC-INDIC DIGIT FOUR
+::s ۵ ::t 5 ::comment EXTENDED ARABIC-INDIC DIGIT FIVE
+::s ۶ ::t 6 ::comment EXTENDED ARABIC-INDIC DIGIT SIX
+::s ۷ ::t 7 ::comment EXTENDED ARABIC-INDIC DIGIT SEVEN
+::s ۸ ::t 8 ::comment EXTENDED ARABIC-INDIC DIGIT EIGHT
+::s ۹ ::t 9 ::comment EXTENDED ARABIC-INDIC DIGIT NINE
+::s ۺ ::t sh ::comment ARABIC LETTER SHEEN WITH DOT BELOW
+::s ۻ ::t d ::comment ARABIC LETTER DAD WITH DOT BELOW
+::s ۼ ::t gh ::comment ARABIC LETTER GHAIN WITH DOT BELOW
+::s ۽ ::t & ::comment ARABIC SIGN SINDHI AMPERSAND
+::s ﷲ ::t allah ::comment ARABIC LIGATURE ALLAH ISOLATED FORM
+
+::s ‌ ::t ::comment ZERO WIDTH NON-JOINER
+::s ‍ ::t ::comment ZERO WIDTH JOINER

uroman/data/romanization-table.txt

@@ -0,0 +1,2019 @@
+
+## European Latin extensions
+# Vowels
+::s Ä ::t Ae
+::s Ö ::t Oe
+::s Ü ::t Ue
+::s Å ::t Aa
+::s Æ ::t Ae
+::s Ø ::t oe
+::s Π::t Oe
+::s ä ::t ae
+::s ö ::t oe
+::s ü ::t ue
+::s å ::t aa
+::s æ ::t ae
+::s ø ::t oe
+::s œ ::t oe
+# Consonants
+::s Ç ::t S
+::s ç ::t s
+::s Ç ::t Ch ::lcode tur
+::s ç ::t ch ::lcode tur
+::s Ş ::t Sh
+::s ş ::t sh
+::s Ș ::t Sh
+::s ș ::t sh
+::s ß ::t ss
+::s Ț ::t Ts
+::s ț ::t ts
+
+# Digraphs
+# ::s ʣ ::t dz
+::s ʤ ::t dzh ::comment Latin small letter dezh digraph
+# ::s ʥ ::t dz
+# ::s ʦ ::t ts
+::s ʧ ::t tsh ::comment Latin small letter tesh digraph
+# ::s ʨ ::t tc
+
+# Miscellaneous
+::s ə ::t e
+
+# English
+::s chr ::t chr ::t-alt kr ::example chromosome, synchronize
+::s Chr ::t Chr ::t-alt Kr ::example Christmas, Chrysler
+::s eight ::t eight ::t-alt eit ::example eight, weight
+::s Eight ::t Eight ::t-alt Eit ::example Eighteen
+::s ight ::t ight ::t-alt ait ::example Knight
+::s gh ::t gh ::t-alt f, ph, "" ::example laugh, daughter
+::s high ::t high ::t-alt hai ::example highlight
+::s High ::t High ::t-alt Hai ::example High School
+::s Isle ::t Isle ::t-alt Ail ::use-only-for-whole-word ::example Isle
+::s Island ::t Island ::t-alt Ailand ::use-only-for-whole-word ::example Island
+::s kn ::t kn ::t-alt n ::use-only-at-start-of-word ::example knowledge
+::s Kn ::t Kn ::t-alt N ::use-only-at-start-of-word ::example Knight
+::s Mc ::t Mc ::t-alt Mac ::use-only-at-start-of-word ::example McNulty
+::s mc ::t mc ::t-alt mac ::use-only-at-start-of-word
+::s oo ::t oo ::t-alt u ::lcode eng ::example Brooklyn; Goose Bay
+::s ph ::t ph ::t-alt f ::example alpha
+::s Ph ::t Ph ::t-alt F ::example Philip
+::s Thom ::t Thom ::t-alt Tom ::use-only-at-start-of-word ::example Thomas, Thompson
+::s tion ::t tion ::t-alt shen ::example
+::s Sean ::t Sean ::t-alt Shawn ::use-only-for-whole-word
+::s ssion ::t ssion ::t-alt shen ::example Sessions
+::s St ::t St ::t-alt Saint ::use-only-for-whole-word
+::s St. ::t St. ::t-alt Saint ::use-only-for-whole-word
+::s Wr ::t Wr ::t-alt R ::example Wren
+::s wr ::t wr ::t-alt r ::example Cartwright
+::s x ::t x ::t-alt ks ::example Mexico
+::s x ::t x ::t-alt gz ::example example, anxiety, exhaust, exit
+
+# French
+::s â ::t a ::t-alt as ::example pâte/paste, pastry
+::s ê ::t e ::t-alt es ::example fête/feast
+::s î ::t i ::t-alt is ::example île/isle
+::s ô ::t o ::t-alt os ::example côte/coast
+::s û ::t u ::t-alt us ::example août/August
+::s eaux ::t eaux ::t-alt o ::example Bordeaux
+::s eau ::t eau ::t-alt o ::example Chateau
+::s auld ::t auld ::t-alt o ::use-only-at-end-of-word ::example Renauld
+::s ault ::t ault ::t-alt o ::use-only-at-end-of-word ::example Renault
+::s oux ::t oux ::t-alt u
+::s ois ::t ois ::t-alt oa ::use-only-at-end-of-word ::example Dubois
+
+# German
+::s Sch ::t Sch ::t-alt Sh
+::s sch ::t sch ::t-alt sh
+::s stein ::t stein ::t-alt shtain
+::s dt ::t dt ::t-alt tt ::use-only-at-end-of-word ::example Schmidt
+
+# Dutch
+::s ij ::t ij ::t-alt ai
+::s Ij ::t Ij ::t-alt Ai
+
+# Latvian
+::s Ā ::t A ::t-alt Aa ::lcode lav
+::s ā ::t a ::t-alt aa ::lcode lav
+::s Ē ::t E ::t-alt Ee ::lcode lav
+::s ē ::t e ::t-alt ee ::lcode lav
+::s Ī ::t I ::t-alt Ii ::lcode lav
+::s ī ::t i ::t-alt ii ::lcode lav
+::s Ū ::t U ::t-alt Uu ::lcode lav
+::s ū ::t u ::t-alt uu ::lcode lav
+::s Ģ ::t G ::t-alt Gj ::lcode lav
+::s ģ ::t g ::t-alt gj ::lcode lav
+::s Ķ ::t K ::t-alt Kj ::lcode lav
+::s ķ ::t k ::t-alt kj ::lcode lav
+::s Ļ ::t L ::t-alt Lj ::lcode lav
+::s ļ ::t l ::t-alt lj ::lcode lav
+::s Ņ ::t N ::t-alt Nj ::lcode lav
+::s ņ ::t n ::t-alt nj ::lcode lav
+::s C ::t C ::t-alt Ts ::lcode lav
+::s c ::t c ::t-alt ts ::lcode lav
+::s Č ::t C ::t-alt Tsh ::lcode lav
+::s č ::t c ::t-alt tsh ::lcode lav
+::s Š ::t Sh ::t-alt s ::lcode lav
+::s š ::t sh ::t-alt s ::lcode lav
+::s Ž ::t Z ::t-alt Zh ::lcode lav
+::s ž ::t z ::t-alt zh ::lcode lav
+
+# Lithuanian
+::s C ::t C ::t-alt Ts ::lcode lit
+::s c ::t c ::t-alt ts ::lcode lit
+::s Č ::t C ::t-alt Tsh ::lcode lit
+::s č ::t c ::t-alt tsh ::lcode lit
+::s Š ::t Sh ::t-alt s ::lcode lit
+::s š ::t sh ::t-alt s ::lcode lit
+::s Ž ::t Z ::t-alt Zh ::lcode lit
+::s ž ::t z ::t-alt zh ::lcode lit
+
+# International Greek (e.g. as used in chemical compounds)
+::s β ::t b
+::s Β ::t B
+::s ϐ ::t b
+
+# Ancient Greek
+::s β ::t b ::lcode grc
+::s Β ::t B ::lcode grc
+::s γγ ::t ng ::lcode grc
+::s γκ ::t nk ::lcode grc
+::s γξ ::t nx ::lcode grc
+::s γχ ::t nch ::lcode grc
+::s ϱ ::t r ::lcode grc
+
+# Pontic Greek
+::s β ::t v ::t-alt b ::lcode pnt
+::s Β ::t V ::t-alt B ::lcode pnt
+::s ϐ ::t v ::t-alt b ::lcode pnt
+
+# Modern Greek (generally the default)
+::s β ::t v ::t-alt b ::lcode ell
+::s Β ::t V ::t-alt B ::lcode ell
+::s ϐ ::t v ::t-alt b ::lcode ell
+::s Ι ::t I
+::s ι ::t i
+::s ί ::t i
+::s ἶ ::t i
+::s Υ ::t Y
+::s υ ::t y
+::s Ρ ::t R
+::s ρ ::t r
+::s ϱ ::t r
+::s Χ ::t Ch ::t-alt Kh
+::s χ ::t ch ::t-alt kh
+::s φ ::t f ::t-alt ph
+::s Φ ::t F ::t-alt Ph
+::s Ντ ::t D
+::s ντ ::t nd ::t-alt d, nt
+# ::s ντζ ::t ntz
+::s Μπ ::t B
+::s μπ ::t b ::use-only-at-start-of-word
+::s μπ ::t mb ::t-alt b, mp ::dont-use-at-start-of-word
+::s λμπ ::t lb
+::s νμπ ::t nb
+::s ρμπ ::t rb
+::s γγ ::t ng
+::s Γκ ::t G
+::s γκ ::t ng ::t-alt g ::dont-use-at-start-of-word
+::s γκ ::t g ::use-only-at-start-of-word
+::s γξ ::t nx ::lcode grc
+::s γχ ::t nch ::lcode grc
+::s ει ::t ei ::t-alt i
+::s Ει ::t Ei ::t-alt I
+::s ευ ::t eu ::t-alt ev ::comment donated by Constantine
+::s Ευ ::t Eu ::t-alt Ev ::comment donated by Constantine
+::s αυ ::t au ::t-alt av
+::s Αυ ::t Au ::t-alt Av
+::s ου ::t ou ::t-alt u
+::s Ου ::t Ou ::t-alt U
+::s ηυ ::t eu
+::s Ηυ ::t Eu
+::s υι ::t ui
+::s Υι ::t Ui
+::s ωυ ::t ou
+::s Ωυ ::t Ou
+::s ͺ ::t ::comment GREEK YPOGEGRAMMENI (U+037A)
+::s ϒ ::t Y ::comment GREEK UPSILON WITH HOOK SYMBOL (U+03D2)
+::s ϓ ::t Y ::comment GREEK UPSILON WITH ACUTE AND HOOK SYMBOL (U+03D3)
+::s ϔ ::t Y ::comment GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL (U+03D4)
+::s ι ::t ::comment GREEK PROSGEGRAMMENI (U+1FBE)
+::s ᾿ ::t ::comment GREEK PSILI (U+1FBF)
+::s ῀ ::t ::comment GREEK PERISPOMENI (U+1FC0)
+::s ` ::t ::comment GREEK VARIA (U+1FEF)
+::s ´ ::t ::comment GREEK OXIA (U+1FFD)
+
+# Glagolitic
+::s Ⰿ ::t M ::comment GLAGOLITIC CAPITAL LETTER MYSLITE (U+2C0F)
+::s Ⱞ ::t M ::comment GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE (U+2C2E)
+::s ⰿ ::t m ::comment GLAGOLITIC SMALL LETTER MYSLITE (U+2C3F)
+::s ⱞ ::t m ::comment GLAGOLITIC SMALL LETTER LATINATE MYSLITE (U+2C5E)
+::s 𞀏 ::t m ::comment COMBINING GLAGOLITIC LETTER MYSLITE (U+1E00F)
+
+# Cyrillic
+::s Г ::t G ::t-alt H ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt h ::comment Cyrillic small ghe
+::s Е ::t E ::t-alt Ye ::comment Cyrillic capital ie
+::s е ::t e ::t-alt ye ::comment Cyrillic small ie
+::s Ё ::t E ::t-alt Yo
+::s ё ::t e ::t-alt yo
+::s Х ::t Kh ::t-alt Ch, H ::comment Cyrillic capital ha
+::s х ::t kh ::t-alt ch, h ::comment Cyrillic small ha
+::s Щ ::t Shch ::t-alt Sh
+::s щ ::t shch ::t-alt sh
+::s Ъ ::t ::comment Cyrillic capital hard sign
+::s ъ ::t ::comment Cyrillic small hard sign
+::s ᲆ ::t ::comment CYRILLIC SMALL LETTER TALL HARD SIGN
+::s Ы ::t Y ::comment Cyrillic capital yeru
+::s ы ::t y ::comment Cyrillic small yeru
+::s Ь ::t ::comment Cyrillic capital soft sign
+::s ь ::t ::comment Cyrillic small soft sign
+::s Ж ::t Zh ::comment Cyrillic capital letter zhe
+::s Ш ::t Sh ::comment Cyrillic capital letter sha
+::s Ч ::t Ch ::comment Cyrillic capital letter che
+::s Џ ::t Dzh ::comment Cyrillic capital letter dzhe
+::s Є ::t Ie ::comment Cyrillic capital letter ie
+::s Ю ::t Yu ::comment Cyrillic capital letter yu
+::s Я ::t Ya ::comment  Cyrillic capital letter ya
+
+::s Ҥ ::t Ng ::comment Cyrillic capital ligature EN GHE
+::s ҥ ::t ng ::comment Cyrillic small ligature EN GHE
+::s Ә ::t e ::comment Cyrillic capital schwa
+::s ә ::t e ::comment Cyrillic small schwa
+::s Ӏ ::t ' ::comment Cyrillic palochka
+::s Ҵ ::t TS ::comment Cyrillic capital ligature te tse, used in Abkhasian
+::s ҵ ::t ts ::comment Cyrillic small ligature te tse, used in Abkhasian
+::s Ӕ ::t AE ::comment Cyrillic capital ligature a ie
+::s ӕ ::t ae ::comment Cyrillic small ligature a ie
+::s ʹ ::t "'" ::comment modifier letter prime
+::s ʺ ::t '"' ::comment modifier letter double prime
+::s ий ::t iy ::dont-use-at-end-of-word
+::s ий ::t y ::use-only-at-end-of-word
+
+::s ᲈ ::t u ::comment CYRILLIC SMALL LETTER UNBLENDED UK ligature ou
+
+# Russian
+::s Г ::t G ::t-alt _NONE_ ::lcode rus ::comment Cyrillic capital letter ghe 
+::s г ::t g ::t-alt _NONE_ ::lcode rus ::comment Cyrillic small letter ghe
+::s Й ::t Y ::t-alt I, J ::lcode rus ::comment Cyrillic capital letter short i 
+::s й ::t y ::t-alt i, j ::lcode rus ::comment Cyrillic small letter short i 
+::s Ц ::t Ts ::t-alt C ::lcode rus ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode rus ::comment Cyrillic small letter tse
+::s Щ ::t Shch ::t-alt _NONE_ ::lcode rus ::comment Cyrillic capital letter shcha 
+::s щ ::t shch ::t-alt _NONE_ ::lcode rus ::comment Cyrillic small letter shcha 
+::s Ѣ ::t E ::t-alt Ie ::lcode rus ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode rus ::comment archaic Cyrillic small letter yat
+::s Е ::t E ::t-alt Ye ::dont-use-at-start-of-word ::lcode rus ::comment Cyrillic capital ie
+::s Е ::t Ye ::t-alt E ::use-only-at-start-of-word ::lcode rus
+::s е ::t e ::t-alt ye ::dont-use-at-start-of-word ::lcode rus ::comment Cyrillic small ie
+::s е ::t ye ::t-alt e ::use-only-at-start-of-word ::lcode rus
+::s ае ::t aye ::lcode rus
+::s а́е ::t aye ::lcode rus
+::s ее ::t eye ::lcode rus
+::s е́е ::t eye ::lcode rus
+::s ие ::t iye ::lcode rus
+::s и́е ::t iye ::lcode rus
+::s ое ::t oye ::lcode rus
+::s о́е ::t oye ::lcode rus
+::s уе ::t uye ::lcode rus
+::s у́е ::t uye ::lcode rus
+::s ье ::t ye ::lcode rus
+::s ъе ::t ye ::lcode rus
+::s Ё ::t Yo ::t-alt E ::lcode rus ::comment Cyrillic capital io
+::s ё ::t yo ::t-alt e ::lcode rus
+::s аё ::t ayo ::lcode rus
+::s а́ё ::t ayo ::lcode rus
+::s её ::t eyo ::lcode rus
+::s е́ё ::t eyo ::lcode rus
+::s иё ::t iyo ::lcode rus
+::s и́ё ::t iyo ::lcode rus
+::s оё ::t oyo ::lcode rus
+::s о́ё ::t oyo ::lcode rus
+::s уё ::t uyo ::lcode rus
+::s у́ё ::t uyo ::lcode rus
+::s ьё ::t yo ::lcode rus
+::s ъё ::t yo ::lcode rus
+::s ий ::t y ::lcode rus
+
+# Ukranian
+::s Г ::t H ::lcode ukr ::comment Ukrainian capital letter he 
+::s г ::t h ::lcode ukr ::comment Ukrainian small letter he
+::s Ґ ::t G ::lcode ukr ::comment Ukrainian capital letter ghe
+::s ґ ::t g ::lcode ukr ::comment Ukrainian small letter ghe
+::s Е ::t E ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic small ie
+::s И ::t Y ::lcode ukr ::comment Ukrainian capital letter i
+::s и ::t y ::lcode ukr ::comment Ukrainian small letter i
+::s Ї ::t Yi ::lcode ukr ::comment Ukrainian capital letter yi
+::s ї ::t yi ::lcode ukr ::comment Ukrainian small letter yi
+::s Й ::t I ::t-alt Y ::lcode ukr ::comment Cyrillic capital letter short i 
+::s й ::t i ::t-alt y ::lcode ukr ::comment Cyrillic small letter short i 
+::s Ц ::t Ts ::t-alt C ::lcode ukr ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode ukr ::comment Cyrillic small letter tse
+::s Щ ::t Shch ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic capital letter shcha 
+::s щ ::t shch ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic small letter shcha 
+::s Ѣ ::t E ::t-alt Ie ::lcode ukr ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode ukr ::comment archaic Cyrillic small letter yat
+::s Иї ::t Yi ::lcode ukr ::comment avoid Yyi
+::s иї ::t yi ::lcode ukr ::comment avoid yyi
+::s ій ::t iy ::lcode ukr
+::s і́й ::t iy ::lcode ukr
+::s ий ::t y ::lcode ukr ::comment Зеленський/Zelensky
+
+# Belarusian
+::s Г ::t H ::t-alt G ::lcode bel ::comment capital letter he 
+::s г ::t h ::t-alt g ::lcode bel ::comment small letter he
+::s Ґ ::t G ::lcode bel ::comment capital letter ghe
+::s ґ ::t g ::lcode bel ::comment small letter ghe
+::s Й ::t J ::t-alt Y ::lcode bel ::comment Cyrillic capital letter short i 
+::s й ::t j ::t-alt y ::lcode bel ::comment Cyrillic small letter short i
+::s Ц ::t Ts ::t-alt C ::lcode bel ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode bel ::comment Cyrillic small letter tse
+::s Щ ::t Shch ::t-alt _NONE_ ::lcode bel ::comment Cyrillic capital letter shcha 
+::s щ ::t shch ::t-alt _NONE_ ::lcode bel ::comment Cyrillic small letter shcha 
+::s Ѣ ::t E ::t-alt Ie ::lcode bel ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode bel ::comment archaic Cyrillic small letter yat
+::s 'я ::t ya ::lcode bel
+::s ’я ::t ya ::lcode bel
+::s 'і ::t i ::lcode bel
+::s ’і ::t i ::lcode bel
+::s Ё ::t Yo ::t-alt E ::lcode bel ::comment Cyrillic capital io
+::s ё ::t yo ::t-alt e ::lcode bel
+::s ёў ::t you ::lcode bel
+::s ий ::t y ::lcode bel
+
+# Serbian
+::s Г ::t G ::t-alt _NONE_ ::lcode srp ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode srp ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt _NONE_ ::lcode srp ::comment Cyrillic capital ha
+::s х ::t h ::t-alt _NONE_ ::lcode srp ::comment Cyrillic small ha
+::s Е ::t E ::t-alt _NONE_ ::lcode srp ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode srp ::comment Cyrillic small ie
+::s Ђ ::t Dj ::lcode srp ::comment Cyrillic capital dje
+::s Љ ::t Lj ::lcode srp ::comment Cyrillic capital lje
+::s Ћ ::t Tsh ::lcode srp ::comment Cyrillic capital tshe
+::s Ж ::t Zh ::lcode srp ::comment Cyrillic capital zhe
+::s Ц ::t C ::t-alt Ts ::lcode srp ::comment Cyrillic capital tse
+::s ц ::t c ::t-alt ts ::lcode srp ::comment Cyrillic capital tse 
+::s Đ ::t Dj ::lcode srp ::comment Latin capital d with stroke
+::s đ ::t dj ::lcode srp ::comment Latin small d with stroke
+::s Ž ::t Zh ::lcode srp ::comment Latin capital z with caron
+::s ž ::t zh ::lcode srp ::comment Latin small z with caron
+::s Ć ::t Tsh ::lcode srp ::comment Latin capital c with acute
+::s ć ::t tsh ::lcode srp ::comment Latin small c with acute
+::s Č ::t Ch ::lcode srp ::comment Latin capital c with caron
+::s č ::t ch ::lcode srp ::comment Latin small c with caron
+::s Š ::t Sh ::lcode srp ::comment Latin capital s with caron
+::s š ::t sh ::lcode srp ::comment Latin small s with caron
+
+::s Г ::t G ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital ha
+::s х ::t h ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small ha
+::s Ц ::t C ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small letter tse
+::s Ч ::t C ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital letter che
+::s ч ::t c ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small letter che
+::s Џ ::t Dz ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital letter dzhe
+::s џ ::t dz ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small letter dzhe
+::s Е ::t E ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small ie
+::s Ш ::t S ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital sha
+::s ш ::t s ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small sha
+::s Ж ::t Z ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital zhe
+::s ж ::t z ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small zhe
+::s Љ ::t Lj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital lje
+::s љ ::t lj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small lje
+::s Њ ::t Nj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital nje
+::s њ ::t nj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small nje
+::s Ђ ::t Dj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital dje
+::s ђ ::t dj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small dje
+::s Ћ ::t C ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital tshe
+::s ћ ::t c ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small tshe
+::s Đ ::t Dj ::lcode srp2 ::comment Latin capital d with stroke
+::s đ ::t dj ::lcode srp2 ::comment Latin small d with stroke
+
+# Montenegrin extension (controversial)
+::s З́ ::t Zj ::lcode srp ::comment Cyrillic capital zje
+::s з́ ::t zj ::lcode srp ::comment Cyrillic small zje
+::s С́ ::t Sj ::lcode srp ::comment Cyrillic capital sje
+::s с́ ::t sj ::lcode srp ::comment Cyrillic small sje
+::s Ź ::t Zj ::lcode srp ::comment Latin capital z with acute
+::s ź ::t zj ::lcode srp ::comment Latin small z with acute
+::s Ś ::t Sj ::lcode srp ::comment Latin capital s with acute
+::s ś ::t sj ::lcode srp ::comment Latin small s with acute
+
+::s З́ ::t Z ::lcode srp2 ::comment Cyrillic capital zje
+::s з́ ::t z ::lcode srp2 ::comment Cyrillic small zje
+::s С́ ::t S ::lcode srp2 ::comment Cyrillic capital sje
+::s с́ ::t s ::lcode srp2 ::comment Cyrillic small sje
+::s Ź ::t Z ::lcode srp2 ::comment Latin capital z with acute
+::s ź ::t z ::lcode srp2 ::comment Latin small z with acute
+::s Ś ::t S ::lcode srp2 ::comment Latin capital s with acute
+::s ś ::t s ::lcode srp2 ::comment Latin small s with acute
+
+# Bulgarian
+::s Г ::t G ::t-alt _NONE_ ::lcode bul ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode bul ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt Kh ::lcode bul ::comment Cyrillic capital letter ha
+::s х ::t h ::t-alt kh ::lcode bul ::comment Cyrillic small letter ha
+::s Ц ::t C ::t-alt Ts ::lcode bul ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt ts ::lcode bul ::comment Cyrillic small letter tse
+::s Щ ::t Sht ::t-alt _NONE_ ::lcode bul ::comment Cyrillic capital letter shcha
+::s щ ::t sht ::t-alt _NONE_ ::lcode bul ::comment Cyrillic small letter shcha
+::s Е ::t E ::t-alt _NONE_ ::lcode bul ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode bul ::comment Cyrillic small ie
+::s Ж ::t Zh ::t-alt Z, J ::lcode bul ::comment Cyrillic capital zhe
+::s ж ::t zh ::t-alt z, j ::lcode bul ::comment Cyrillic small zhe
+::s Й ::t I ::t-alt Y, J ::lcode bul ::comment Cyrillic capital letter short i
+::s й ::t i ::t-alt y, j ::lcode bul ::comment Cyrillic short letter short i
+::s Ю ::t Yu ::t-alt U, Ju, Iu ::lcode bul ::comment Cyrillic capital letter yu
+::s ю ::t yu ::t-alt u, ju, iu ::lcode bul ::comment Cyrillic small letter yu
+::s Ъ ::t U ::t-alt A ::lcode bul ::comment Cyrillic capital letter hard sign
+::s ъ ::t u ::t-alt a ::lcode bul ::comment Cyrillic capital letter hard sign
+::s Ѣ ::t E ::t-alt Ie ::lcode bul ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode bul ::comment archaic Cyrillic small letter yat
+::s Ѫ ::t U ::lcode bul ::comment archaic Cyrillic capital letter yus
+::s ѫ ::t u ::lcode bul ::comment archaic Cyrillic small letter yus
+::s ИЯ ::t IA ::lcode bul ::use-only-at-end-of-word
+::s ия ::t ia ::lcode bul ::use-only-at-end-of-word
+
+::s Ž ::t Zh ::lcode bul ::comment Latin capital z with caron
+::s ž ::t zh ::lcode bul ::comment Latin small z with caron
+::s Č ::t Ch ::lcode bul ::comment Latin capital c with caron
+::s č ::t ch ::lcode bul ::comment Latin small c with caron
+::s Š ::t Sh ::lcode bul ::comment Latin capital s with caron
+::s š ::t sh ::lcode bul ::comment Latin small s with caron
+::s Ŝ ::t Sht ::lcode bul ::comment Latin capital s with circumflex
+::s ŝ ::t sht ::lcode bul ::comment Latin small s with circumflex
+::s Û ::t Yu ::t-alt U, Ju, Iu ::lcode bul ::comment Latin capital u with circumflex
+::s û ::t yu ::t-alt u, ju, iu ::lcode bul ::comment Latin small u with circumflex
+::s  ::t Ya ::t-alt _NONE_ ::lcode bul ::comment Latin capital a with circumflex
+::s â ::t ya ::t-alt _NONE_ ::lcode bul ::comment Latin small a with circumflex
+::s Ŭ ::t U ::t-alt A ::lcode bul ::comment Latin capital u with breve (for hard sign)
+::s ŭ ::t u ::t-alt a ::lcode bul ::comment Latin small u with breve (for hard sign)
+::s Ǎ ::t U ::t-alt A ::lcode bul ::comment Latin capital a with caron (for hard sign)
+::s ǎ ::t u ::t-alt a ::lcode bul ::comment Latin small a with caron (for hard sign)
+
+# Macedonian
+::s Г ::t G ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic small ghe
+::s Х ::t H ::lcode mkd ::comment Cyrillic capital ha
+::s х ::t h ::lcode mkd ::comment Cyrillic small ha
+::s Ц ::t C ::t-alt Ts ::lcode mkd ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt ts ::lcode mkd ::comment Cyrillic small letter tse
+::s Џ ::t Dzh ::t-alt Dj, Dz ::lcode mkd ::comment Cyrillic capital letter dzhe
+::s џ ::t dzh ::t-alt dj, dz ::lcode mkd ::comment Cyrillic small letter dzhe
+::s Е ::t E ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic small ie
+::s Ž ::t Zh ::lcode mkd ::comment Latin capital z with caron
+::s ž ::t zh ::lcode mkd ::comment Latin small z with caron
+::s Č ::t Ch ::lcode mkd ::comment Latin capital c with caron
+::s č ::t ch ::lcode mkd ::comment Latin small c with caron
+::s Š ::t Sh ::lcode mkd ::comment Latin capital s with caron
+::s š ::t sh ::lcode mkd ::comment Latin small s with caron
+::s Ǵ ::t Gj ::lcode mkd
+::s ǵ ::t gj ::lcode mkd
+::s Đ ::t Gj ::lcode mkd
+::s đ ::t gj ::lcode mkd
+::s Ẑ ::t Dz ::lcode mkd
+::s ẑ ::t dz ::lcode mkd
+::s J̌ ::t J ::lcode mkd
+::s ǰ ::t j ::lcode mkd
+::s L̂ ::t Lj ::lcode mkd
+::s l̂ ::t lj ::lcode mkd
+::s N̂ ::t Nj ::lcode mkd
+::s n̂ ::t nj ::lcode mkd
+::s Ḱ ::t Kj ::lcode mkd
+::s ḱ ::t kj ::lcode mkd
+::s Ć ::t Kj ::lcode mkd
+::s ć ::t kj ::lcode mkd
+::s D̂ ::t Dzh ::lcode mkd
+::s d̂ ::t dzh ::lcode mkd
+
+::s Г ::t G ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital ha
+::s х ::t h ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small ha
+::s Ц ::t C ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small letter tse
+::s Ч ::t C ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital letter che
+::s ч ::t c ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small letter che
+::s Џ ::t D ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital letter dzhe
+::s џ ::t d ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small letter dzhe
+::s Е ::t E ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small ie
+::s Ш ::t S ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital sha
+::s ш ::t s ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small sha
+::s Ѓ ::t G ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital gje
+::s ѓ ::t g ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small gje
+::s Ж ::t Z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital zhe
+::s ж ::t z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small zhe
+::s Ѕ ::t Z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital dze
+::s ѕ ::t z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small dze
+::s Ќ ::t K ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital kje
+::s ќ ::t k ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small kje
+::s Љ ::t L ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital lje
+::s љ ::t l ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small lje
+::s Њ ::t N ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital nje
+::s њ ::t n ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small nje
+::s Ž ::t Z ::lcode mkd2 ::comment Latin capital z with caron
+::s ž ::t z ::lcode mkd2 ::comment Latin small z with caron
+::s Č ::t C ::lcode mkd2 ::comment Latin capital c with caron
+::s č ::t c ::lcode mkd2 ::comment Latin small c with caron
+::s Š ::t S ::lcode mkd2 ::comment Latin capital s with caron
+::s š ::t s ::lcode mkd2 ::comment Latin small s with caron
+::s Ǵ ::t G ::lcode mkd2
+::s ǵ ::t g ::lcode mkd2
+::s Đ ::t G ::lcode mkd2
+::s đ ::t g ::lcode mkd2
+::s Ẑ ::t D ::lcode mkd2
+::s ẑ ::t d ::lcode mkd2
+::s J̌ ::t J ::lcode mkd2
+::s ǰ ::t j ::lcode mkd2
+::s L̂ ::t L ::lcode mkd2
+::s l̂ ::t l ::lcode mkd2
+::s N̂ ::t N ::lcode mkd2
+::s n̂ ::t n ::lcode mkd2
+::s Ḱ ::t K ::lcode mkd2
+::s ḱ ::t k ::lcode mkd2
+::s Ć ::t K ::lcode mkd2
+::s ć ::t k ::lcode mkd2
+::s D̂ ::t D ::lcode mkd2
+::s d̂ ::t d ::lcode mkd2
+
+# Kazakh
+::s Ә ::t A ::lcode kaz
+::s ә ::t a ::lcode kaz
+::s Г ::t G ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small ghe
+::s Ғ ::t G ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital ghe with stroke
+::s ғ ::t g ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small ghe with stroke
+::s Е ::t E ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small ie
+::s Ё ::t Yo ::t-alt _NONE_ ::lcode kaz
+::s ё ::t yo ::t-alt _NONE_ ::lcode kaz
+::s Х ::t H ::t-alt X ::lcode kaz ::comment Cyrillic capital ha
+::s х ::t h ::t-alt x ::lcode kaz ::comment Cyrillic small ha
+::s Һ ::t H ::lcode kaz ::comment Cyrillic capital shha
+::s һ ::t h ::lcode kaz ::comment Cyrillic small shha
+::s Қ ::t Q ::t-alt K ::lcode kaz 
+::s қ ::t q ::t-alt k ::lcode kaz
+::s Ц ::t Ts ::t-alt C ::lcode kaz ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode kaz ::comment Cyrillic small letter tse
+::s Щ ::t Sh ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital letter shcha
+::s щ ::t sh ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small letter shcha
+::s У ::t U ::t-alt Y ::lcode kaz
+::s у ::t u ::t-alt y ::lcode kaz
+::s уы ::t wy ::lcode kaz
+::s Ж ::t J ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital zhe
+::s ж ::t j ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small zhe
+::s Ю ::t Yw ::t-alt Yuw, Yiw ::lcode kaz ::comment Cyrillic capital letter yu
+::s ю ::t yw ::t-alt yuw, yiw ::lcode kaz ::comment Cyrillic small letter yu
+
+# Kyrgyz
+::s Г ::t G ::t-alt _NONE_ ::lcode kir ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode kir ::comment Cyrillic small ghe
+::s Е ::t E ::t-alt Ye ::lcode kir ::comment Cyrillic capital ie
+::s е ::t e ::t-alt ye ::lcode kir ::comment Cyrillic small ie
+::s Ё ::t Yo ::t-alt _NONE_ ::lcode kir
+::s ё ::t yo ::t-alt _NONE_ ::lcode kir
+::s Х ::t Kh ::t-alt X, H ::lcode kir ::comment Cyrillic capital ha
+::s х ::t kh ::t-alt x, h ::lcode kir ::comment Cyrillic small ha
+::s Ж ::t Zh ::t-alt J ::lcode kir ::comment Cyrillic capital zhe
+::s ж ::t zh ::t-alt j ::lcode kir ::comment Cyrillic small zhe
+::s Й ::t Y ::t-alt I ::lcode kir ::comment Cyrillic capital letter short i 
+::s й ::t y ::t-alt i ::lcode kir ::comment Cyrillic small letter short i
+::s Ц ::t Ts ::t-alt C ::lcode kir ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode kir ::comment Cyrillic small letter tse
+::s Ң ::t Ng ::lcode kir
+::s ң  ::t ng ::lcode kir
+::s Ө ::t O ::t-alt Oe ::lcode kir
+::s ө ::t o ::t-alt oe ::lcode kir
+::s Ү ::t U ::t-alt Y, Ue ::lcode kir
+::s ү ::t u ::t-alt y, ue ::lcode kir
+::s Ы ::t I ::t-alt Y ::lcode kir
+::s ы ::t i ::t-alt y ::lcode kir
+::s йы ::t yi ::lcode kir
+::s ый ::t iy ::lcode kir
+
+# Ossetian
+::s ийы ::t iy ::lcode oss
+
+# Gothic
+::s 𐌴 ::t e ::comment Gothic letter aihvus
+::s 𐌹 ::t i ::comment Gothic letter eis
+::s 𐍇 ::t x ::comment Gothic letter iggws
+
+# Runic
+::s ᛫ ::t " " ::comment Runic single punctuation, used as word separator
+::s ᛬ ::t . ::comment Runic multiple punctuation, used as sentence separator
+
+# Ogham
+::s ᚁ ::t b ::comment Ogham letter Beith	
+::s ᚂ ::t l ::comment Ogham letter Luis	
+::s ᚃ ::t f ::comment Ogham letter Fearn	
+::s ᚄ ::t s ::comment Ogham letter Sail	
+::s ᚅ ::t n ::comment Ogham letter Nion	
+::s ᚋ ::t m ::comment Ogham letter Muin
+::s ᚌ ::t g ::comment Ogham letter Gort
+::s ᚍ ::t v ::t-alt ng ::comment Ogham letter nGéadal
+::s ᚎ ::t z ::comment Ogham letter Straif
+::s ᚏ ::t r ::comment Ogham letter Ruis
+::s ᚆ ::t h ::t-alt j ::comment Ogham letter Uath	
+::s ᚇ ::t d	::comment Ogham letter Dair	
+::s ᚈ ::t t ::comment Ogham letter Tinne	
+::s ᚉ ::t k ::comment Ogham letter Coll	
+::s ᚊ ::t q ::t-alt kw ::comment Ogham letter Ceirt
+::s ᚐ ::t a ::comment Ogham letter Ailm
+::s ᚑ ::t o	::comment Ogham letter Onn
+::s ᚒ ::t u ::comment Ogham letter Úr
+::s ᚓ ::t e ::comment Ogham letter Eadhadh
+::s ᚔ ::t i ::comment Ogham letter Iodhadh
+::s ᚚ ::t p ::comment Ogham letter Peith
+# Additional Ogham letters (outside standard alphabet)
+::s ᚕ ::t eo ::t-alt ea ::comment Ogham additional letter Éabhadh
+::s ᚖ ::t oi ::t-alt oe ::comment Ogham additional letter Ór
+::s ᚗ ::t ui ::t-alt ua ::comment Ogham additional letter Uilleann
+::s ᚘ ::t p ::t-alt io ::comment Ogham additional letter Ifín 
+::s ᚙ ::t ch ::t-alt x, ai ::comment Ogham additional letter Eamhancholl
+::s   ::t " " ::comment Ogham space mark
+::s ᚛ ::t "" ::comment Ogham feather mark
+::s ᚜ ::t "" ::comment Ogham feather mark
+
+# Georgian
+::s ა ::t a ::comment Georgian letter an
+::s ე ::t e ::comment Georgian letter en
+::s ი ::t i ::comment Georgian letter in
+::s ო ::t o ::comment Georgian letter on
+::s უ ::t u ::comment Georgian letter un
+::s ჱ ::t ey ::comment archaic Georgian letter he
+::s ჲ ::t i ::comment archaic Georgian letter hie
+::s ჳ :::t w ::comment archaic Georgian letter we
+::s ჴ ::t q ::comment archaic Georgian letter har
+::s ჵ ::t o ::comment archaic Georgian letter hoe
+::s ჶ ::t f ::comment Georgian letter fi (Greek phi)
+::s ჷ ::t e ::comment Georgian letter yn (schwa)
+::s ჸ ::t a ::comment Georgian letter elifi
+::s ჹ ::t g ::comment Georgian letter gan
+::s ჺ ::t ' ::comment Georgian letter ain
+::s ჼ ::t n ::comment Georgian letter nar
+::s ჽ ::t e ::comment Georgian letter aen
+::s ჾ ::t ::comment Georgian letter hard sign
+::s ჿ ::t w ::comment Georgian letter labial sign
+
+::s Ⴚ ::t TS ::comment GEORGIAN CAPITAL LETTER CAN
+::s ც ::t ts ::comment GEORGIAN LETTER CAN
+::s Ც ::t TS ::comment GEORGIAN MTAVRULI CAPITAL LETTER CAN
+::s ⴚ ::t ts ::comment GEORGIAN SMALL LETTER CAN
+::s Ⴜ ::t TS ::comment GEORGIAN CAPITAL LETTER CIL
+::s წ ::t ts ::comment GEORGIAN LETTER CIL
+::s Წ ::t TS ::comment GEORGIAN MTAVRULI CAPITAL LETTER CIL
+::s ⴜ ::t ts ::comment GEORGIAN SMALL LETTER CIL
+::s Ⴛ ::t DZ ::comment GEORGIAN CAPITAL LETTER JIL
+::s ძ ::t dz ::comment GEORGIAN LETTER JIL
+::s Ძ ::t DZ ::comment GEORGIAN MTAVRULI CAPITAL LETTER JIL
+::s ⴛ ::t dz ::comment GEORGIAN SMALL LETTER JIL
+::s Ⴟ ::t J ::comment GEORGIAN CAPITAL LETTER JHAN
+::s ჯ ::t j ::comment GEORGIAN LETTER JHAN
+::s Ჯ ::t J ::comment GEORGIAN MTAVRULI CAPITAL LETTER JHAN
+::s ⴟ ::t j ::comment GEORGIAN SMALL LETTER JHAN
+
+
+::s Ⴀ ::t A ::comment Georgian capital letter an
+::s Ⴄ ::t E ::comment Georgian capital letter en
+::s Ⴈ ::t I ::comment Georgian capital letter in
+::s Ⴍ ::t O ::comment Georgian capital letter on
+::s Ⴓ ::t U ::comment Georgian capital letter un
+::s Ⴡ ::t EY ::comment archaic Georgian capital letter he
+::s Ⴢ ::t I ::comment archaic Georgian capital letter hie
+::s Ⴣ :::t W ::comment archaic Georgian capitel letter we
+::s Ⴤ ::t Q ::comment archaic Georgian capital letter har
+::s Ⴥ ::t O ::comment archaic Georgian capital letter hoe
+::s Ⴧ ::t E ::comment archaic Georgian capital letter yn (schwa)
+::s Ⴭ ::t E ::comment archaic Georgian capital letter aen
+
+::s Ა ::t A ::comment Georgian Mtavruli capital letter an
+::s Ე ::t E ::comment Georgian Mtavruli capital letter en
+::s Ი ::t I ::comment Georgian Mtavruli capital letter in
+::s Ო ::t O ::comment Georgian Mtavruli capital letter on
+::s Უ ::t U ::comment Georgian Mtavruli capital letter un
+::s Ჱ ::t EY ::comment archaic Georgian Mtavruli capital letter he
+::s Ჲ ::t I ::comment archaic Georgian Mtavruli capital letter hie
+::s Ჳ :::t W ::comment archaic Georgian Mtavruli capital letter we
+::s Ჴ ::t Q ::comment archaic Georgian Mtavruli capital letter har
+::s Ჵ ::t O ::comment archaic Georgian Mtavruli capital letter hoe
+::s Ჶ ::t F ::comment Georgian Mtavruli capital letter fi (Greek phi)
+::s Ჷ ::t E ::comment Georgian Mtavruli capital letter yn (schwa)
+::s Ჸ ::t A ::comment Georgian Mtavruli capital letter elifi
+::s Ჹ ::t G ::comment Georgian Mtavruli capital letter gan
+::s Ჺ ::t ' ::comment Georgian Mtavruli capital letter ain
+::s Ჽ ::t E ::comment Georgian Mtavruli capital letter aen
+::s Ჾ ::t ::comment Georgian Mtavruli capital letter hard sign
+::s Ჿ ::t W ::comment Georgian Mtavruli capital letter labial sign
+
+::s ⴀ ::t a ::comment Georgian small letter an
+::s ⴄ ::t e ::comment Georgian small letter en
+::s ⴈ ::t i ::comment Georgian small letter in
+::s ⴍ ::t o ::comment Georgian small letter on
+::s ⴓ ::t u ::comment Georgian small letter un
+::s ⴡ ::t ey ::comment archaic Georgian small letter he
+::s ⴢ ::t i ::comment archaic Georgian small letter hie
+::s ⴣ :::t w ::comment archaic Georgian small letter we
+::s ⴤ ::t q ::comment archaic Georgian small letter har
+::s ⴥ ::t o ::comment archaic Georgian small letter hoe
+::s ⴧ ::t e ::comment Georgian small letter yn (schwa)
+::s ⴭ ::t e ::comment Georgian small letter aen
+
+# Armenian
+::s Ա ::t A ::comment Armenian capital letter ayb
+::s ա ::t a ::comment Armenian small letter ayb
+::s ՠ ::t a ::comment ARMENIAN SMALL LETTER TURNED AYB (CHECK)
+::s Ե ::t E ::comment Armenian capital letter ech ::dont-use-at-start-of-word
+::s ե ::t e ::comment Armenian small letter ech ::dont-use-at-start-of-word
+::s Ե ::t Ye ::comment Armenian capital letter ech ::use-only-at-start-of-word
+::s ե ::t ye ::comment Armenian small letter ech ::use-only-at-start-of-word
+::s Է ::t E ::comment Armenian capital letter eh
+::s է ::t e ::comment Armenian small letter eh
+::s Ը ::t E ::comment Armenian capital letter et
+::s ը ::t e ::comment Armenian small letter et
+::s Ի ::t I ::comment Armenian capital letter ini
+::s ի ::t i ::comment Armenian small letter ini
+::s Յ ::t Y ::comment Armenian capital letter yi
+::s յ ::t y ::comment Armenian small letter yi
+::s ֈ ::t y ::comment ARMENIAN SMALL LETTER YI WITH STROKE (CHECK)
+::s Ո ::t Vo ::comment Armenian capital letter vo ::use-only-at-start-of-word
+::s ո ::t vo ::comment Armenian small letter vo ::use-only-at-start-of-word
+::s Ո ::t O ::comment Armenian capital letter vo ::dont-use-at-start-of-word
+::s ո ::t o ::comment Armenian small letter vo ::dont-use-at-start-of-word
+::s Ւ ::t W ::comment Armenian capital letter yiwn
+::s ւ ::t w ::comment Armenian small letter yiwn
+::s Օ ::t O ::comment Armenian capital letter oh
+::s օ ::t o ::comment Armenian small letter oh
+::s Խ ::t Kh ::comment Armenian capital letter xeh
+::s խ ::t kh ::comment Armenian small letter xeh
+
+::s Ժ ::t Zh ::comment Armenian capital letter zhe
+::s Ղ ::t Gh ::comment Armenian capital letter ghad
+::s Ճ ::t Tch ::comment Armenian capital letter cheh
+::s ճ ::t tch ::comment Armenian small letter cheh
+::s Շ ::t Sh ::comment Armenian capital letter sha
+::s Չ ::t Ch ::comment Armenian capital letter cha
+::s Ջ ::t J ::comment Armenian capital letter jheh
+::s ջ ::t j ::comment Armenian small letter jheh
+::s Վ ::t V ::comment Armenian capital letter vew
+::s վ ::t v ::comment Armenian small letter vew
+::s Ձ ::t Dz ::comment Armenian capital letter ja
+::s ձ ::t dz ::comment Armenian small letter ja
+::s Ծ ::t Ts ::comment Armenian capital letter ca
+::s ծ ::t ts ::comment Armenian small letter ca
+::s Ք ::t K ::t-alt Q ::comment Armenian capital letter keh - sometimes romanized as K' or Q
+::s ք ::t k ::t-alt q ::comment Armenian small letter keh - sometimes romanized as k' or q
+
+::s են ::t en ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s եմ ::t em ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s ենք ::t enk ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s ես ::t es ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s եք ::t ek ::use-only-for-whole-word ::comment exception (auxiliary verb)
+
+::s և ::t ev ::comment Armenian small ligature ech yiwn
+::s ՈՒ ::t U ::comment Armenian capital vo+yiwn
+::s Ու ::t U ::comment Armenian capital/small vo+yiwn
+::s ու ::t u ::comment Armenian small vo+wywn
+
+::s իւ ::t yu 
+
+## Japanese
+# Katakana
+::s シ ::t shi
+::s チ ::t chi
+::s フ ::t fu
+::s ジ ::t ji
+::s ヂ ::t ji
+::s ヅ ::t zu
+::s シャ ::t sha
+::s シュ ::t shu
+::s ショ ::t sho
+::s チャ ::t cha
+::s チェ ::t che
+::s チュ ::t chu
+::s チョ ::t cho
+::s ジャ ::t ja
+::s ジュ ::t ju
+::s ジョ ::t jo
+::s ジェ ::t je
+::s ヂャ ::t ja
+::s ヂュ ::t ju
+::s ヂョ ::t jo
+::s フェ ::t fe
+::s ヴェ ::t ve
+::s フィ ::t fi
+::s ウィ ::t wi
+::s ヴィ ::t vi
+::s ティ ::t ti
+::s ディ ::t di
+::s ッ ::t (__SOKUON__) ::comment katakana double following consonant
+::s ー ::t (__CHOONPU__) ::comment katakana prolonged sound mark
+::s 𛅤 ::t i ::comment KATAKANA LETTER SMALL WI
+::s 𛅥 ::t e ::comment KATAKANA LETTER SMALL WE
+::s 𛅦 ::t o ::comment KATAKANA LETTER SMALL WO
+# Hiragana
+::s し ::t shi
+::s ち ::t chi
+::s つ ::t tsu
+::s ふ ::t fu
+::s を ::t o
+::s じ ::t ji
+::s ぢ ::t ji
+::s づ ::t zu
+::s しゃ ::t sha
+::s しゅ ::t shu
+::s しょ ::t sho
+::s ちゃ ::t cha
+::s ちゅ ::t chu
+::s ちょ ::t cho
+::s じゃ ::t ja
+::s じゅ ::t ju
+::s じょ ::t jo
+::s ぢゃ ::t ja
+::s ぢゅ ::t ju
+::s ぢょ ::t jo
+::s 𛅐 ::t i ::comment HIRAGANA LETTER SMALL WI
+::s 𛅑 ::t e ::comment HIRAGANA LETTER SMALL WE
+::s 𛅒 ::t o ::comment HIRAGANA LETTER SMALL WO
+::s っ ::t (__SOKUON__) ::comment hiragana double following consonant
+::s 々 ::t ² ::comment ideographic iteration mark ::annotation repetition-sign
+
+::s フ ::t fu ::t-alt f
+::s キ ::t ki ::t-alt k
+::s ク ::t ku ::t-alt k
+::s ラ ::t ra ::t-alt la
+::s リ ::t ri ::t-alt li
+::s ル ::t ru ::t-alt lu, l, r
+::s レ ::t re ::t-alt le
+::s ロ ::t ro ::t-alt lo
+::s ム ::t mu ::t-alt m ::example キム = Kim
+::s シ ::t shi ::t-alt si ::example メキシコ = meksiko (Mexico)
+::s ス ::t su ::t-alt s
+::s ト ::t to ::t-alt t
+::s ツ ::t tsu ::t-alt tu, ts ::example シュルツ = Schultz
+
+::s ㋿ ::t Reiwa ::comment SQUARE ERA NAME REIWA 
+
+# Chinese
+::s 邦 ::t bang ::t-alt bon, bum, bun, pon
+::s 鲍 ::t bao ::t-alt bow
+::s 堡 ::t bao ::t-alt berg, burg, bourg, burgh
+::s 贝 ::t bei ::t-alt ber
+::s 本 ::t ben ::t-alt bern, bon, bourn, burn
+::s 彼得 ::t bide ::t-alt peter, pet
+::s 伯 ::t bo ::t-alt ber
+::s 波 ::t bo ::t-alt po
+::s 布 ::t bu ::t-alt b
+::s 策 ::t ce ::t-alt tze, tzer
+::s 曾 ::t ceng ::t-alt tzen, zen
+::s 彻 ::t che ::t-alt tche
+::s 茨 ::t ci ::t-alt ts, tz, z
+::s 兹 ::t ci ::t-alt ds, dz, tz, z, zi
+::s 蒂 ::t di ::t-alt ti, tti
+::s 丁 ::t ding ::t-alt din, tin
+::s 顿 ::t dun ::t-alt ton
+::s 多 ::t duo ::t-alt do, dor, to
+::s 尔 ::t er ::t-alt l, le, ll, r
+::s 弗 ::t fu ::t-alt f, fer, pher, v, ver, vir
+::s 夫 ::t fu ::t-alt f, v, v
+::s 福 ::t fu ::t-alt faw, for, ford
+::s 哥 ::t ge ::t-alt go, co
+::s 戈 ::t ge ::t-alt go
+::s 各 ::t ge ::t-alt go, co
+::s 赫 ::t he ::t-alt ch, che, cher, ge
+::s 华 ::t hua ::t-alt ver, wa, war, wer ::example Washington
+::s 怀 ::t huai ::t-alt whi, wi, wy
+::s 惠 ::t hui ::t-alt wha, whea
+::s 基 ::t ji ::t-alt ki, chi
+::s 吉 ::t ji ::t-alt gi, gui
+::s 加 ::t jia ::t-alt ca, ga, ka ::example Canada
+::s 杰 ::t jie ::t-alt ger
+::s 金 ::t jin ::t-alt kin, gin
+::s 斤 ::t jin ::t-alt zin
+::s 康 ::t kang ::t-alt con, corn
+::s 考 ::t kao ::t-alt cow, cour
+::s 克 ::t ke ::t-alt k, che, cher
+::s 科 ::t ke ::t-alt ko
+::s 拉 ::t la ::t-alt ra ::example Tirana
+::s 朗 ::t lang ::t-alt lon, ron
+::s 赖 ::t lai ::t-alt ri
+::s 劳 ::t lao ::t-alt low
+::s 勒 ::t lei ::t-alt ler
+::s 伦 ::t lun ::t-alt lon, ran, ron
+::s 里 ::t li ::t-alt ri
+::s 利 ::t li ::t-alt ri ::example Ferrari
+::s 隆 ::t long ::t-alt lon, lum, lund
+::s 罗 ::t luo ::t-alt l, lo, lu, ro, row, ru
+::s 洛 ::t luo ::t-alt lo, low, ro
+::s 默 ::t mo ::t-alt mer
+::s 纳 ::t na ::t-alt ne, ner
+::s 珀 ::t po ::t-alt per
+::s 奇 ::t qi ::t-alt chi, dge, ge, tch
+::s 齐 ::t qi ::t-alt tsi, zi
+::s 乔 ::t qiao ::t-alt jo
+::s 青 ::t qing ::t-alt tsing
+::s 琼 ::t qiong ::t-alt jon, jum, jun
+::s 瑟 ::t se ::t-alt the
+::s 什 ::t shen ::t-alt sh
+::s 圣 ::t sheng ::t-alt san, sao, saint
+::s 斯 ::t si ::t-alt s, rth, th ::example Alaska
+::s 索 ::t suo ::t-alt tho
+::s 特 ::t te ::t-alt t
+::s 翁 ::t weng ::t-alt on
+::s 沃 ::t wo ::t-alt ver, vo, war, wer
+::s 乌 ::t wu ::t-alt ou, u
+::s 希 ::t xi ::t-alt chi, hi, shi
+::s 西 ::t xi ::t-alt s, si
+::s 锡 ::t xi ::t-alt ci, si, thi, zi
+::s 夏 ::t xia ::t-alt ha, cha, cia, sha, tia
+::s 香 ::t xiang ::t-alt chan, cham
+::s 歇 ::t xie ::t-alt she
+::s 谢 ::t xie ::t-alt che, she
+::s 辛 ::t xin ::t-alt cin, sen, sin, sing, sun, zen
+::s 欣 ::t xin ::t-alt hin, shin
+::s 休 ::t xiu ::t-alt hu, hue
+::s 修 ::t xiu ::t-alt ciu, siu, thew, tiu
+::s 许 ::t xu ::t-alt hue, schue
+::s 逊 ::t xun ::t-alt son
+::s 耶 ::t ye ::t-alt yer, ier
+::s 泽 ::t ze ::t-alt ser
+::s 扎 ::t zha ::t-alt za
+::s 詹 ::t zhan ::t-alt ja, jam, jan, jen, jon
+::s 治 ::t zhi ::t-alt ge ::example George
+
+## Numbers
+# Chinese and Japanese numbers
+::s 零 ::num 0
+::s 〇 ::num 0
+::s 一 ::num 1
+::s 二 ::num 2
+::s 三 ::num 3
+::s 四 ::num 4
+::s 五 ::num 5
+::s 六 ::num 6
+::s 七 ::num 7
+::s 八 ::num 8
+::s 九 ::num 9
+::s 十 ::num 10
+::s 百 ::num 100
+::s 千 ::num 1000
+::s 万 ::num 10000
+::s 萬 ::num 10000
+::s 亿 ::num 100000000
+::s 億 ::num 100000000
+::s 兆 ::num 1000000000000
+::s 京 ::num 10000000000000000
+
+# numbers in non-number words (to be exptended)
+::s 一贯 ::t yiguan ::comment consistent
+
+::s 红十字会 ::t hongshizihui ::comment Red Cross
+
+::s 百度 ::t baidu ::comment Baidu (company)
+::s 百分 ::t baifen ::comment percent
+::s 百合 ::t baihe ::comment lily
+::s 百货 ::t baihuo ::comment general merchandise
+::s 百科 ::t baike ::comment encyclopedia
+::s 百老汇 ::t bailaohui
+::s 百灵 ::t bailing
+::s 百慕大 ::t baimuda
+::s 百日咳 ::t bairike
+::s 百色市 ::t baiseshi
+::s 百事可乐 ::t baishikele ::comment Pepsi Cola
+::s 百無 ::t baiwu
+::s 百香 ::t baixiang
+::s 百姓 ::t baixing
+::s 百叶 ::t baiye
+::s 百色 ::t bose
+::s 杨百翰 ::t yangbaihan ::comment Brigham Young
+
+::s 北京 ::t beijing
+::s 京都 ::t jingdou
+::s 东京 ::t dongjing
+::s 京胡 ::t jinghu
+::s 南京 ::t nangjing
+::s 普京 ::t pujing ::comment Putin
+::s 東京 ::t dongjing ::comment Tokyo
+::s 京兆 ::t jingzhao
+
+::s ㎢ ::t km²
+::s ㎥ ::t m³
+::s ㎝ ::t cm
+
+## Indian
+# see mostly under UnicodeDataOverwrite.txt
+
+# Malayalam
+::s ൗ ::t au ::comment MALAYALAM AU LENGTH MARK
+
+# Tamil
+::s ட ::t d ::comment most commonly d, but t when word-initial or in a doubled consonant
+::s ஃப ::t f ::comment h+p=f
+::s ஃஜ ::t z ::comment h+j=z
+
+# Myanmar/Burmese
+# ::s ့ ::t ::comment dot below, denotes creaky tone 
+# ::s း ::t ::comment visarga, denotes high tone
+::s ၌ ::t -nai ::comment locative
+::s ၍ ::t -jwe ::comment completed
+::s ၎ ::t legau ::comment aforementioned
+::s ၏ ::t -i ::comment genetive
+
+# Lao
+::s ັ ::t a ::comment vowel sign mai kan
+::s ົ ::t o ::comment vowel sign mai kon
+::s ູ ::t uu ::comment vowel sign uu
+::s ຽ ::t y ::comment semivowel sign nyo
+::s ຼ ::t l ::comment semivowel sign lo
+::s ລ ::t l ::comment lo loot
+::s ຣ ::t l ::comment lo ling
+::s ໝ ::t m ::comment ho mo
+::s ໜ ::n ::comment ho no
+::s ຢ ::t y ::comment yo
+::s ໍ ::t oo ::comment niggahita (possibly also nasal -m in final position)
+::s ໆ ::t ² ::comment Lao ko la ::annotation repetition-sign
+::s ຯ ::t ...  ::comment Lao ellipsis
+
+# Thai
+::s ออ ::t o
+::s อั ::t a
+::s อิ ::t i
+::s ๆ ::t ² ::comment Thai character maiyamok ::annotation repetition-sign
+
+# Khmer
+::s ័ ::t "" ::comment Khmer samyok sannya: indicates deviation from the general rules of pronunciation
+::s ៏ ::t "" ::comment Khmer sign ahsda: denotes stressed intonation in some single-consonant words
+::s ៍ ::t "" ::comment Khmer sign toandakhiat: indicates that the base character is not pronounced
+::s ៌ ::t "" ::comment Khmer sign robat: a diacritic historically corresponding to the repha form of ra in Devanagari
+::s ប៉ ::t pa ::comment Khmer ba + musĕkâtônd -> pa
+::s ៗ ::t ² ::comment Khmer sign lek too ::annotation repetition-sign
+
+## Semitic languages
+# Arabic
+::s و ::t w  ::comment Arabic letter waw  ::t-alt o, u  ::lcode ara
+::s ء ::t ' ::comment hamza 
+::s ٔ ::t ' ::comment hamza above
+::s ٕ ::t ' ::comment hamza below
+::s ع ::t ' ::comment ain
+::s آ ::t a ::comment alef madda
+::s ٓا ::t a ::comment Arabic maddah above plus alef (presumably an ill-formed version of آ; found 1 instance in Urdu text)
+::s إ ::t i ::comment alef with hamza below
+::s ٱ ::t a ::comment alef wasla ::comment typically indicates liaison with preceding word 
+::s ة ::t a ::comment teh marbuta
+::s ۃ ::t a ::comment teh marbuta goal ::comment Used in Punjabi, Sindhi. Different from plain 'teh marbuta'?
+::s ي ::t y ::comment Arabic yeh
+::s ى ::t a ::comment alef maksura
+::s ﻯ ::t a ::comment alef maksura isolated form
+::s ﻰ ::t a ::comment alef maksura final form
+::s ﯨ ::t a ::comment Uighur Kazach Kirghiz alef maksura initial form
+::s ﯩ ::t a ::comment Uighur Kazach Kirghiz alef maksura medial form
+::s ٰ ::t a  ::comment Arabic letter superscript alef
+::s ـ ::t   ::comment tatweel (filler)
+::s َ ::t a ::comment fatha ("-a")
+::s ُ ::t u ::comment damma ("-u")
+::s ِ ::t i ::comment kasra ("-i")
+::s ْ ::t   ::comment sukun (no vowel)
+::s ۡ ::t  ::comment small high dotless head of khah; like sukun (no vowel); used in Kashmiri, Assamese
+::s ً ::t   ::comment fathatan ("-an")
+::s اً ::t an ::comment alef + fathatan
+::s ٌ ::t   ::comment dammatan ("-un")
+::s ٍ ::t   ::comment kasratan ("-in")
+::s ّ ::t   ::comment shadda (consonant doubler)
+::s ڃ ::t ny ::comment Arabic letter nyeh U+0683 (used in Sindhi (snd))
+::s ڄ ::t dy ::comment Arabic letter dyeh U+0684 (used in Sindhi (snd))
+::s ۾ ::t men ::comment Sindhi postposition men
+::s ؑ ::t alayhe wasallam ::comment "upon him be peace"
+::s ﷴ ::t mohammad ::comment "Mohammad"
+::s ﷸ ::t wasallam ::comment "and peace"
+::s ﷺ ::t sallallahou alayhe wasallam ::comment "prayer of God be upon him and his family and peace"
+
+::s ࣓ ::t waw ::comment ARABIC SMALL LOW WAW
+::s ࣔ ::t al-rub ::comment ARABIC SMALL HIGH WORD AR-RUB
+::s ࣕ ::t s ::comment ARABIC SMALL HIGH SAD
+::s ࣖ ::t ' ::comment ARABIC SMALL HIGH AIN
+::s ࣗ ::t q ::comment ARABIC SMALL HIGH QAF
+::s ࣘ ::t n ::comment ARABIC SMALL HIGH NOON WITH KASRA
+::s ࣙ ::t n ::comment ARABIC SMALL LOW NOON WITH KASRA
+::s ࣚ ::t al-thalatha ::comment ARABIC SMALL HIGH WORD ATH-THALATHA
+::s ࣛ ::t al-sajda ::comment ARABIC SMALL HIGH WORD AS-SAJDA
+::s ࣜ ::t al-nisf ::comment ARABIC SMALL HIGH WORD AN-NISF
+::s ࣝ ::t sakta ::comment ARABIC SMALL HIGH WORD SAKTA
+::s ࣞ ::t qif ::comment ARABIC SMALL HIGH WORD QIF
+::s ࣟ ::t waqfa ::comment ARABIC SMALL HIGH WORD WAQFA
+::s ࣠ ::t ::comment ARABIC SMALL HIGH FOOTNOTE MARKER (CHECK)
+::s ࣡ ::t ::comment ARABIC SMALL HIGH SIGN SAFHA (CHECK)
+::s ࣢ ::t ::comment ARABIC DISPUTED END OF AYAH (CHECK)
+
+# Farsi
+::s ی ::t i ::t-alt y ::comment Contributed by Nima
+::s ای ::t i ::t-alt ai ::use-only-at-start-of-word ::comment Contributed by Nima
+::s هٔ ::t eye ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s و ::t v ::t-alt o, u ::lcode fas ::comment Arabic letter waw
+::s ض ::t z ::t-alt d ::lcode fas ::comment Contributed by Marjan
+::s ث ::t s ::t-alt th ::lcode fas ::comment Contributed by Marjan
+::s ذ ::t z ::t-alt th ::lcode fas ::comment Contributed by Nima
+::s ع ::t a ::t-alt ' ::lcode fas ::comment Contributed by Nima
+::s عا ::t a ::lcode fas ::comment Contributed by Nima
+::s عی ::t i ::t-alt iy ::lcode fas ::comment Contributed by Nima
+::s عو ::t u ::t-alt o, av ::lcode fas ::comment Contributed by Nima
+::s چ ::t ch ::t-alt tch, tsh ::lcode fas ::comment Contributed by Nima
+::s ه ::t e ::t-alt h ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s ‌ ::t "" ::t-alt " " ::lcode fas ::comment source is character "zero-width non-joiner" (U+200C); Contributed by Nima
+::s غ ::t gh ::t-alt g ::lcode fas
+::s آئی ::t ai ::t-alt ae ::lcode fas
+::s ائی ::t ai ::t-alt ae ::lcode fas
+::s آئو ::t au ::t-alt ao ::lcode fas
+::s ائو ::t au ::t-alt ao ::lcode fas
+
+# Kashmiri (so far: educated guesses)
+::s ٖ ::t a ::comment Arabic subscript alef U+0656
+::s ٗ ::t u ::comment Arabic inverted damma U+0657
+::s ۚ ::t j ::comment Arabic small high jeem U+06DA
+::s ۪ ::t   ::comment Arabic emtpy centre low stop U+06EA
+::s ۬ ::t   ::comment Arabic rounded high stop with filled center U+06EC 
+
+# Pashto
+::s ٙ ::t e ::comment Arabic zwarakay
+::s ځ ::t z ::t-alt dz ::comment Pashto letter zim; Arabic letter "hah with hamza above"
+::s څ ::t ts ::t-alt c ::comment Pashto letter tsim; Arabic letter "h with three dots above"
+::s ګ ::t g ::comment Pashto letter gaf; Arabic letter "kaf with ring"
+::s ڼ ::t n ::comment Arabic letter "noon with ring"
+::s ږ ::t g ::t-alt z, zh, j ::comment pronunciation varies regionally
+::s ښ ::t kh ::t-alt sh ::comment pronunciation varies regionally
+::s ه ::t h ::t-alt a ::lcode pus
+::s ۀ ::t e ::lcode pus ::comment Arabic letter "heh with yeh above"
+::s و ::t w ::t-alt o, u ::lcode pus
+::s ی ::t ay ::t-alt y ::lcode pus
+::s وی ::t wy ::t-alt oy, uy ::lcode pus
+::s ای ::t ay ::lcode pus
+::s ۍ ::t ay ::lcode pus
+::s ئ ::t ay ::t-alt y ::lcode pus
+::s ژ ::t zh ::t-alt z ::lcode pus ::comment [ʒ]
+::s ض ::t z ::t-alt d ::lcode pus
+::s ث ::t s ::lcode pus ::t-alt th ::comment Arabic letter theh (unvoiced th/θ)
+::s ذ ::t z ::lcode pus ::t-alt th ::comment Arabic letter thal (voiced th/ð)
+
+# Hebrew
+::s ב ::t v ::comment Hebrew letter bet  ::t-alt b
+::s כ ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s ך ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s פ ::t f ::comment Hebrew letter pe  ::t-alt p
+::s ש ::t sh  ::comment Hebrew letter shin  ::t-alt s
+::s ו ::t v  ::comment Hebrew letter vav  ::t-alt o, u
+::s ח ::t ch ::comment Hebrew letter het  ::t-alt h ::use-alt-in-pointed
+::s ק ::t q ::t-alt k ::use-alt-in-pointed
+::s וֹ ::t o
+::s וּ ::t u
+::s קְוָ ::t qva ::t-alt kva  ::use-alt-in-pointed
+::s י ::t y
+::s יּ ::t y
+::s יָּ ::t ya
+::s ײ ::t yy ::comment Hebrew ligature Yiddish double Yod (CHECK)
+::s ׯ ::t yyy ::comment HEBREW YOD TRIANGLE (CHECK)
+::s ע ::t '
+::s ִי ::t i ::t-alt iy ::use-alt-in-pointed
+::s ֵי ::t e
+::s ִיּ ::t iy
+::s ִיָּ ::t iya
+::s ױ ::t oy
+::s א ::t a ::t-alt '
+::s אָ ::t a
+::s ֹא ::t o
+::s אַ ::t 'a
+::s אֲ ::t 'a
+::s אֶ ::t e
+::s אֱ ::t e
+::s פ ::t f
+::s פּ ::t p
+::s פַּ ::t pa
+::s פְּ ::t pe ::t-alt p ::use-alt-in-pointed 
+::s שׁ ::t sh
+::s שָׁ ::t sha
+::s שָּׁ ::t sha ::comment ?
+::s שְׁ ::t she ::t-alt sh ::use-alt-in-pointed
+::s שֶׁ ::t she
+::s שִׁ ::t shi
+::s שֻׁ ::t shu
+::s שׂ ::t s
+::s שָׂ ::t sa
+::s שְׂ ::t s ::t-alt se ::use-alt-in-pointed
+::s כּ ::t k
+::s כֶּ ::t ke
+::s כֹּ ::t ko
+::s בּ ::t b
+::s בַּ ::t ba
+::s בָּ ::t ba
+::s בְּ ::t be ::t-alt b ::use-alt-in-pointed
+::s בֶּ ::t be
+::s תּ ::t t
+::s תַּ ::t ta
+::s תֵּ ::t te
+::s תִּ ::t ti
+::s דָּ ::t da
+::s דְּ ::t de ::t-alt d ::use-alt-in-pointed
+::s גּ ::t g
+::s לֵּ ::t le
+::s ד׳ ::t dh
+::s ג׳ ::t j
+::s ת׳ ::t th
+::s ז׳ ::t zh
+::s חַ ::t ach ::comment furtive patah ::use-only-at-end-of-word
+::s עַ ::t a' ::comment furtive patah ::use-only-at-end-of-word
+::s הַּ ::t ah ::comment furtive patah ::use-only-at-end-of-word
+::s ַ ::t a ::comment Hebrew point patah
+::s ֲ ::t a ::comment Hebrew point hataf patah (hataf = reduced)
+::s ֳ ::t o ::comment Hebrew point hataf qamats
+::s ָ ::t a ::comment Hebrew point qamats ::t-alt o ::use-alt-in-pointed
+::s ֶ ::t e ::comment Hebrew point segol
+::s ֱ ::t e ::comment Hebrew point hataf segol (hataf = reduced)
+::s ְ ::t e ::comment Hebrew point sheva ::t-alt "" ::use-alt-in-pointed
+::s ֵ ::t e ::comment Hebrew point tsere
+::s ִ ::t i ::comment Hebrew point hiriq
+::s ֹ ::t o ::comment Hebrew point holam
+::s ֻ ::t u ::comment Hebrew point qubuts
+# ::s ּ ::t "" ::comment Hebrew point dagesh or mapiq
+
+# Yiddish
+::s א ::t a ::lcode yid ::comment called "silent" alef
+::s אי ::t y ::lcode yid
+::s איי ::t ey ::lcode yid
+::s או ::t u ::lcode yid
+::s אוי ::t oy ::lcode yid
+::s אַ ::t a ::lcode yid
+::s אָ ::t o ::lcode yid
+::s ב ::t b ::lcode yid
+::s בֿ ::t v ::lcode yid
+::s דזש ::t dzh ::lcode yid
+::s ו ::t u ::lcode yid
+::s וּ ::t u ::lcode yid
+::s וֹ ::t o ::lcode yid
+::s װ ::t v ::lcode yid
+::s ווא ::t wa ::lcode yid
+::s וואַ ::t wa ::lcode yid
+::s ווע ::t we ::lcode yid
+::s ווי ::t wi ::lcode yid
+::s וואוי ::t wo ::lcode yid
+::s וי ::t oy ::lcode yid
+::s זש ::t zh ::lcode yid
+::s ח ::t ch ::lcode yid
+::s טש ::t tsh ::lcode yid
+::s יִ::t i ::lcode yid
+::s יי ::t ey ::lcode yid ::comment maybe "yi" at beginning of word
+::s ײַ ::t ay ::lcode yid
+::s כּ ::t k ::lcode yid
+::s כ ::t ch ::lcode yid
+::s ך ::t ch ::lcode yid
+::s ע ::t e ::lcode yid
+::s פּ ::t p ::lcode yid
+::s פֿ ::t f ::lcode yid
+::s ף ::t f ::lcode yid ::comment sometimes p
+::s ק ::t k ::lcode yid
+::s ת ::t s ::lcode yid
+
+# Syriac/Aramaic (should be vetted by expert)
+::s ܰ ::t a  ::comment Syriac pthaha above
+::s ܲ ::t a  ::comment Syriac pthaha dotted
+::s ܳ ::t aa  ::comment Syriac zqapha above
+::s ܴ ::t aa  ::comment Syriac zqapha below
+::s ܵ ::t aa  ::comment Syriac zqapha dotted
+::s ܶ ::t e  ::comment Syriac rbasa above
+::s ܷ ::t e  ::comment Syriac rbasa below
+::s ܿ ::t o  ::comment Syriac rwaha
+::s ܸ ::t e  ::comment Syriac dotted zlama horizontal
+::s ܹ ::t e  ::comment Syriac dotted zlama angular
+::s ܺ ::t i  ::comment Syriac hbasa above
+::s ܝܺ ::t i  ::comment Syriac yudh + hbasa above
+::s ܼ ::t u  ::comment Syriac hbasa-esasa dotted
+::s ܽ ::t o  ::comment Syriac esasa above
+::s ܾ ::t u  ::comment Syriac esasa below
+::s ݇ ::t ""  ::comment Syriac oblique line above; indication of a silent letter
+
+::s ܖ ::t d  ::comment Syriac letter dotless dalath rish; ambiguous form for undifferentiated early dalath/rish
+::s ܜ ::t t  ::comment Syriac letter teth garshuni; used in Garshuni documents
+::s ܒ݂ ::t v  ::comment Syriac beth + rukkakha
+::s ܒ�� ::t v  ::comment Syriac beth + ring-below
+::s ܓ݂ ::t g  ::comment Syriac gammal + rukkakha [IPA: ɣ]
+::s ܓ̥ ::t g  ::comment Syriac gammal + ring-below [IPA: ɣ]
+::s ܕ݂ ::t d  ::comment Syriac dalath + rukkakha [IPA: ð]
+::s ܕ̥ ::t d  ::comment Syriac dalath + ring-below [IPA: ð]
+::s ܟ݂ ::t kh  ::comment Syriac kaph + rukkakha [IPA: x]
+::s ܟ̥ ::t kh  ::comment Syriac kaph + ring-below [IPA: x]
+::s ܦ݂ ::t f  ::comment Syriac pe + rukkakha
+::s ܦ̥ ::t f  ::comment Syriac pe + ring-below
+::s ܦ݁ ::t p  ::comment Syriac pe + qushshaya
+::s ܬ݂ ::t th  ::comment Syriac taw + rukkakha [IPA: θ]
+::s ܬ̥ ::t th  ::comment Syriac taw + ring-below [IPA: θ]
+
+::s ܄ ::t : ::comment Syriac sublinear colon; used at the end of verses of supplicationscolon skewed left
+::s ܆ ::t , ::comment Syriac colon skewed left; marks a dependent clause
+::s ܇ ::t , ::comment Syriac colon skewed right; marks the end of a subdivision of the apodosis, or latter part of a Biblical verse
+
+# Uzbek
+::s ʻ ::t ' ::comment modifies pronunciation of preceding "o" and "g"
+::s ʼ ::t ' ::comment glottal stop (tutuq belgisi)
+
+# Uyghur
+::s ئا ::t a ::lcode uig
+::s ە ::t e ::lcode uig
+::s ئې ::t e ::lcode uig ::latinplus ë
+::s ې ::t e ::lcode uig ::latinplus ë
+::s ئە ::t e ::lcode uig
+::s يە ::t e ::lcode uig
+::s ئى ::t i ::lcode uig
+::s ى ::t i ::lcode uig
+::s ئو ::t o ::lcode uig
+::s و ::t o ::lcode uig
+::s ئۇ ::t u ::lcode uig
+::s ۇ ::t u ::lcode uig
+::s چ ::t ch ::t-alt q ::lcode uig
+::s خ ::t x ::lcode uig
+::s ژ ::t zh ::lcode uig
+::s ئۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ئۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۋ ::t w ::lcode uig
+
+# Maldivian
+::s ް ::t ::comment thaana sukun
+::s ަ ::t a ::comment thaana abafili
+::s ާ ::t aa ::comment thaana aabaafili
+::s ި ::t i ::comment thaana ibifili
+::s ީ ::t ee ::comment thaana eebeefili
+::s ު ::t u ::comment thaana ubufili
+::s ޫ ::t oo ::comment thaana ooboofili
+::s ެ ::t e ::comment thaana ebefili
+::s ޭ ::t ey ::comment thaana eybeyfili
+::s ޮ ::t o ::comment thaana obofili
+::s ޯ ::t oa ::comment thaana oaboafili
+
+# Canadian syllabics (Inuktitut)
+::s ᑊ ::t p ::comment syllable final
+::s ᐟ ::t t ::comment syllable final
+::s ᐠ ::t k ::comment syllable final
+::s ᐨ ::t c ::comment syllable final
+::s ᒼ ::t m ::comment syllable final
+::s ᐣ ::t n ::comment syllable final
+::s ᐢ ::t s ::comment syllable final
+::s ᐧ ::t y ::comment syllable final
+::s ᐤ ::t w ::comment syllable final
+::s ᐦ ::t h ::comment syllable final
+::s ᕽ ::t hk ::comment syllable final
+::s ᓫ ::t l ::comment syllable final
+::s ᕑ ::t r ::comment syllable final
+
+# Mongolian
+::s ᢅ ::t ::comment MONGOLIAN LETTER ALI GALI BALUDA (CHECK) indicates assimilation
+::s ᢆ ::t ::comment MONGOLIAN LETTER ALI GALI THREE BALUDA (CHECK) indicates assimilation
+
+# Tibetan
+::s ྅ ::t ::comment TIBETAN MARK PALUTA (CHECK) indicates assimilation
+
+## Punctuation
+# delete
+::s ¿ ::t "" ::comment inverted question mark
+::s ¡ ::t "" ::comment inverted exclamation mark
+# decompose double-punctuation
+::s ‼ ::t !!
+::s ⁇ ::t ??
+::s ⁉ ::t !?
+::s ⁈ ::t ?!
+# preserve
+::s ′ ::t ′
+::s ∩ ::t ∩
+::s ‡ ::t ‡
+# Cyrillic
+::s ⁙ ::t . ::comment five dot punctuation
+# Amharic/Ethiopian
+::s ። ::t .
+::s ፣ ::t ,
+::s ፤ ::t ;
+::s ፥ ::t :
+::s ፧ ::t ? ::comment Ethiopic question mark
+::s ፡ ::t " " ::comment Ethiopic wordspace
+::s ፦ ::t :  ::comment Ethiopic preface colon
+# Ethiopic wordspace often appropriated for other purposes:
+::s ፡፡ ::t .
+::s ፡- ::t :
+::s "፡ " ::t ", "
+::s ቸ ::t cha ::comment Ethiopic syllable ca
+::s ቹ ::t chu ::comment Ethiopic syllable cu
+::s ቺ ::t chi ::comment Ethiopic syllable ci
+::s ቻ ::t chaa ::comment Ethiopic syllable caa
+::s ቼ ::t chee ::comment Ethiopic syllable cee
+::s ች ::t che ::comment Ethiopic syllable ce
+::s ቾ ::t cho ::comment Ethiopic syllable co
+::s ሠ ::t sa ::comment Ethiopic syllable sza
+::s ሡ ::t su ::comment Ethiopic syllable szu
+::s ሢ ::t si ::comment Ethiopic syllable szi
+::s ሣ ::t saa ::comment Ethiopic syllable szaa
+::s ሤ ::t see::comment Ethiopic syllable szee
+::s ሥ ::t se ::comment Ethiopic syllable sze
+::s ሦ ::t so ::comment Ethiopic syllable szo
+::s ጠ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጡ ::t tu ::comment Ethiopic syllable thu with ejective 't'
+::s ጢ ::t ti ::comment Ethiopic syllable thi with ejective 't'
+::s ጣ ::t taa ::comment Ethiopic syllable thaa with ejective 't'
+::s ጤ ::t tee ::comment Ethiopic syllable thee with ejective 't'
+::s ጥ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጦ ::t to ::comment Ethiopic syllable tho with ejective 't'
+
+# Devanagari (Hindi etc.)
+::s । ::t . ::comment danda
+::s ॥ ::t . ::comment double danda
+::s ৷ ::t . ::comment Bengali currency numerator four; used as danda
+::s ॰ ::t . ::comment Devanagari abbreviation sign
+# Bengali
+::s ৽ ::t . ::comment BENGALI ABBREVIATION SIGN
+::s ৾ ::t ::comment BENGALI SANDHI MARK (CHECK)
+# Gurmukhi
+::s ੶ ::t . ::comment GURMUKHI ABBREVIATION SIGN
+# Oriya/Odia (India)
+::s ୤ ::t . ::comment danda (deprecated, should use Devanagari danda ।)
+::s ୥ ::t . ::comment double danda (deprecated, should use Devanagari double danda ॥)
+# Tibetan
+::s ། ::t ,
+::s །: ::t :
+::s ༏ ::t ;
+::s ༎ ::t .
+::s ༑ ::t ,  ::comment Tibetan mark run chen spungs shad
+::s ༼ ::t (  ::comment Tibetan open roof punctuation
+::s ༽ ::t )  ::comment Tibetan close roof punctuation
+::s ༈ ::t ""  ::comment Tibetan mark srbul shad
+::s 【 ::t [  ::comment left black lenticular bracket
+::s 】 ::t ]  ::comment right black lenticular bracket
+::s ༄ ::t ""  ::comment Tibetan head mark
+::s ༄༅ ::t ""  ::comment Tibetan head mark
+::s ༆ ::t ""  ::comment Tibetan head mark
+# Myanmar/Burmese
+::s ၊ ::t ,
+::s ။ ::t .
+Khmer
+::s ៖ ::t ; ::comment Khmer sign camnuc pii kuuh
+::s ។ ::t . ::comment Khmer sign khan
+# Arabic
+::s ، ::t ,
+::s ؛ ::t ;
+::s ٬ ::t ,
+::s ۔ ::t .
+::s ؟ ::t ?
+::s ٪ ::t %
+::s ٫ ::t , ::comment Arabic decimal separator
+::s ۽ ::t & ::comment Arabic sign Sindhi ampersand
+# Aramaic
+::s ܀ ::t .
+::s ܂ ::t .
+# Hebrew
+::s ־ ::t - ::comment maqaf
+# Armenian
+::s ։ ::t .
+::s ՝ ::t , ::comment Armenian comma
+# Chinese
+::s ， ::t ", "
+::s 、 ::t ", "
+::s 。 ::t ". "
+::s ！ ::t "! "
+::s ？ ::t "? "
+::s 「 ::t ' "'
+::s 」 ::t '" '
+::s 《 ::t ' "'
+::s 》 ::t '" '
+::s （ ::t " ("
+::s ） ::t ") "
+::s ； ::t ;
+::s ： ::t ": "
+::s ︰ ::t ": "
+::s － ::t -
+::s ／ ::t /
+::s ＝ ::t =
+::s ～ ::t ~
+::s ＆ ::t &
+::s ＜ ::t <
+::s ＞ ::t >
+::s ％ ::t %
+::s ＿ ::t _ ::comment FULLWIDTH LOW LINE (U+FF3F)
+::s ｛ ::t { ::comment FULLWIDTH LEFT CURLY BRACKET (U+FF5B)
+::s ｝ ::t } ::comment FULLWIDTH RIGHT CURLY BRACKET (U+FF5D)
+::s 　 ::t " " ::comment ideographic space
+# Japanese
+::s 『 ::t ' "'
+::s 』 ::t '" '
+::s ・ ::t " "  ::comment Katakana middle dot; separates name elements such as first and last name
+# N'ko
+::s ߽ ::t . ::comment NKO DANTAYALAN used to abbreviate units of measure
+# Medefaidrin
+::s 𖺗 ::t , ::comment MEDEFAIDRIN COMMA
+::s 𖺘 ::t . ::comment MEDEFAIDRIN FULL STOP
+# Khitan
+::s 𖿤 ::t ::comment KHITAN SMALL SCRIPT FILLER
+
+# Symbols
+::s ∞ ::t ∞  ::comment infinity
+::s ­ ::t   ::comment soft hyphen; used to indicate preferred line breaks; remove
+::s ֊ ::t -  ::comment Armenian hyphen; map to regular hyphen-minus
+::s ᐩ ::t +  ::comment Canadian syllabics final plus; map to regular plus
+::s ﹐ ::t ,  ::comment small comma; map to regular comma
+::s ˚ ::t °  ::comment ring above; map to degree sign
+::s ⇒ ::t ⇒ ::comment rightwards double arrow
+::s † ::t †  ::comment dagger
+::s • ::t •  ::comment bullet
+::s ℃ ::t °C  ::comment degree Celsius; split into 2 characters
+::s ℉ ::t °F  ::comment degree Fahrenheit; split into 2 characters
+::s ― ::t ―  ::comment horizontal bar
+::s ˇ ::t ˇ  ::comment caron (sometimes apparently used for "Arabic vowel sign small v above" U+065A, e.g. in Gilaki language (glk))
+::s ″ ::t ″  ::comment double prime
+::s ﴾ ::t (  ::comment ornate left parenthesis
+::s ﴿ ::t )  ::comment ornate right parenthesis
+::s 〔 ::t [  ::comment left tortoise shell bracket
+::s 〕 ::t ]  ::comment right tortoise shell bracket
+::s ﹝ ::t (  ::comment small left tortoise shell bracket
+::s ﹞ ::t )  ::comment small left tortoise shell bracket
+::s ¦ ::t ¦ ::comment BROKEN BAR (U+00A6)
+::s ¨ ::t ::comment DIAERESIS (U+00A8)
+::s ¯ ::t ::comment MACRON (U+00AF)
+::s ¸ ::t ::comment CEDILLA (U+00B8)
+::s Ƿ ::t W ::comment LATIN CAPITAL LETTER WYNN (U+01F7)
+::s ˘ ::t ::comment BREVE (U+02D8)
+::s ˛ ::t ::comment OGONEK (U+02DB)
+::s ˜ ::t ~ ::comment SMALL TILDE (U+02DC)
+::s ̒ ::t ::comment COMBINING TURNED COMMA ABOVE (U+0312)
+::s ̔ ::t ::comment COMBINING REVERSED COMMA ABOVE (U+0314)
+::s ̜ ::t ::comment COMBINING LEFT HALF RING BELOW (U+031C)
+::s ̧ ::t ::comment COMBINING CEDILLA (U+0327)
+::s ̫ ::t ::comment COMBINING INVERTED DOUBLE ARCH BELOW (U+032B)
+::s ̲ ::t ::comment COMBINING LOW LINE (U+0332)
+::s ̳ ::t ::comment COMBINING DOUBLE LOW LINE (U+0333)
+::s ̹ ::t ::comment COMBINING RIGHT HALF RING BELOW (U+0339)
+::s ̺ ::t ::comment COMBINING INVERTED BRIDGE BELOW (U+033A)
+::s ̿ ::t ::comment COMBINING DOUBLE OVERLINE (U+033F)
+::s ͅ ::t ::comment COMBINING GREEK YPOGEGRAMMENI (U+0345)
+::s ͑ ::t ::comment COMBINING LEFT HALF RING ABOVE (U+0351)
+::s ͗ ::t ::comment COMBINING RIGHT HALF RING ABOVE (U+0357)
+::s ͚ ::t ::comment COMBINING DOUBLE RING BELOW (U+035A)
+::s ͜ ::t ::comment COMBINING DOUBLE BREVE BELOW (U+035C)
+::s ͝ ::t ::comment COMBINING DOUBLE BREVE (U+035D)
+::s ͞ ::t ::comment COMBINING DOUBLE MACRON (U+035E)
+::s ͟ ::t ::comment COMBINING DOUBLE MACRON BELOW (U+035F)
+::s ͠ ::t ::comment COMBINING DOUBLE TILDE (U+0360)
+
+::s ‐ ::t - ::comment HYPHEN (U+2010)
+::s ‗ ::t ‗ ::comment DOUBLE LOW LINE (U+2017)
+::s ‵ ::t ‵ ::comment REVERSED PRIME (U+2035)
+::s ‶ ::t ‶ ::comment REVERSED DOUBLE PRIME (U+2036)
+::s ‸ ::t ‸ ::comment CARET (U+2038)
+::s ‽ ::t ?! ::comment INTERROBANG (U+203D)
+::s ‾ ::t ‾ ::comment OVERLINE (U+203E)
+::s ‿ ::t ‿ ::comment UNDERTIE (U+203F)
+::s ⁂ ::t ⁂ ::comment ASTERISM (U+2042)
+::s ⁎ ::t * ::comment LOW ASTERISK (U+204E)
+::s ⁏ ::t ; ::comment REVERSED SEMICOLON (U+204F)
+::s ⁔ ::t ⁔ ::comment INVERTED UNDERTIE (U+2054)
+::s ⁝ ::t ⁝ ::comment TRICOLON (U+205D)
+::s   ::t " " ::comment MEDIUM MATHEMATICAL SPACE (U+205F)
+::s ₋ ::t - ::comment SUBSCRIPT MINUS (U+208B)
+::s ⃩ ::t ::comment COMBINING WIDE BRIDGE ABOVE (U+20E9)
+
+::s ﹔ ::t ; ::comment SMALL SEMICOLON (U+FE54)
+::s ﹕ ::t : ::comment SMALL COLON (U+FE55)
+::s ﹛ ::t { ::comment SMALL LEFT CURLY BRACKET (U+FE5B)
+::s ﹜ ::t } ::comment SMALL RIGHT CURLY BRACKET (U+FE5C)
+::s ﹠ ::t & ::comment SMALL AMPERSAND (U+FE60)
+::s ﹡ ::t * ::comment SMALL ASTERISK (U+FE61)
+::s ﹣ ::t - ::comment SMALL HYPHEN-MINUS (U+FE63)
+
+::s ℈ ::t ℈ ::comment SCRUPLE (U+2108)
+::s ℟ ::t ℟ ::comment RESPONSE (U+211F)
+::s ℣ ::t ℣ ::comment VERSICLE (U+2123)
+::s ℽ ::t ℽ ::comment DOUBLE-STRUCK SMALL GAMMA (U+213D)
+::s ℾ ::t ℾ ::comment DOUBLE-STRUCK CAPITAL GAMMA (U+213E)
+::s ⅋ ::t ⅋ ::comment TURNED AMPERSAND (U+214B)
+::s ⅍ ::t A/S::comment AKTIESELSKAB (U+214D)
+
+::s ⑃ ::t ⑃ ::comment OCR INVERTED FORK (U+2443)
+::s ⑊ ::t \\ ::comment OCR DOUBLE BACKSLASH (U+244A)
+::s ⟮ ::t ( ::comment MATHEMATICAL LEFT FLATTENED PARENTHESIS (U+27EE)
+::s ⟯ ::t ) ::comment MATHEMATICAL RIGHT FLATTENED PARENTHESIS (U+27EF)
+::s ⸨ ::t (( ::comment LEFT DOUBLE PARENTHESIS (U+2E28)
+::s ⸩ ::t )) ::comment RIGHT DOUBLE PARENTHESIS (U+2E29)
+
+# kavyka indicates alternative reading
+::s ᷶ ::t ::comment COMBINING KAVYKA ABOVE RIGHT (U+1DF6)
+::s ᷷ ::t ::comment COMBINING KAVYKA ABOVE LEFT (U+1DF7)
+::s ⹅ ::t ::comment INVERTED LOW KAVYKA (U+2E45)
+::s ⹆ ::t ::comment INVERTED LOW KAVYKA WITH KAVYKA ABOVE (U+2E46)
+::s ⹇ ::t ::comment LOW KAVYKA (U+2E47)
+::s ⹈ ::t ::comment LOW KAVYKA WITH DOT (U+2E48)
+::s ꙾ ::t ::comment CYRILLIC KAVYKA (U+A67E)
+
+# Braille
+::s ⠁ ::t a
+::s ⠃ ::t b
+::s ⠉ ::t c
+::s ⠙ ::t d
+::s ⠑ ::t e
+::s ⠋ ::t f
+::s ⠛ ::t g
+::s ⠓ ::t h
+::s ⠊ ::t i
+::s ⠚ ::t j
+::s ⠅ ::t k
+::s ⠇ ::t l
+::s ⠍ ::t m
+::s ⠝ ::t n
+::s ⠕ ::t o
+::s ⠏ ::t p
+::s ⠟ ::t q
+::s ⠗ ::t r
+::s ⠎ ::t s
+::s ⠞ ::t t
+::s ⠥ ::t u
+::s ⠧ ::t v
+::s ⠺ ::t w
+::s ⠭ ::t x
+::s ⠽ ::t y
+::s ⠵ ::t z
+
+::s ⠜ ::t ae
+::s ⠪ ::t oe
+::s ⠳ ::t ue
+::s ⠷ ::t a ::comment à
+::s ⠡ ::t a ::comment â
+::s ⠿ ::t e ::comment é
+::s ⠮ ::t e ::comment è
+::s ⠣ ::t e ::comment ê
+::s ⠫ ::t e ::comment ë
+::s ⠩ ::t i ::comment î
+::s ⠻ ::t i ::comment ï
+::s ⠹ ::t o ::comment ô
+::s ⠾ ::t u ::comment ù
+::s ⠱ ::t u ::comment û
+
+::s ⠡ ::t au ::lcode deu
+::s ⠌ ::t aeu ::lcode deu
+::s ⠹ ::t ch ::lcode deu
+::s ⠩ ::t ei ::lcode deu
+::s ⠣ ::t eu ::lcode deu
+::s ⠬ ::t ie ::lcode deu
+::s ⠱ ::t sch ::lcode deu
+::s ⠮ ::t ss ::lcode deu
+::s ⠾ ::t st ::lcode deu
+
+::s ⠠⠠ ::t "" ::comment start of word all-caps mode
+# ::s ⠠⠁ ::t A
+# ::s ⠠⠃ ::t B
+# ::s ⠠⠉ ::t C
+# ::s ⠠⠙ ::t D
+# ::s ⠠⠑ ::t E
+# ::s ⠠⠋ ::t F
+# ::s ⠠⠛ ::t G
+# ::s ⠠⠓ ::t H
+# ::s ⠠⠊ ::t I
+# ::s ⠠⠚ ::t J
+# ::s ⠠⠅ ::t K
+# ::s ⠠⠇ ::t L
+# ::s ⠠⠍ ::t M
+# ::s ⠠⠝ ::t N
+# ::s ⠠⠕ ::t O
+# ::s ⠠⠏ ::t P
+# ::s ⠠⠟ ::t Q
+# ::s ⠠⠗ ::t R
+# ::s ⠠⠎ ::t S
+# ::s ⠠⠞ ::t T
+# ::s ⠠⠥ ::t U
+# ::s ⠠⠧ ::t V
+# ::s ⠠⠺ ::t W
+# ::s ⠠⠭ ::t X
+# ::s ⠠⠽ ::t Y
+# ::s ⠠⠵ ::t Z
+
+::s ⠼⠁ ::t 1
+::s ⠼⠃ ::t 2
+::s ⠼⠉ ::t 3
+::s ⠼⠙ ::t 4
+::s ⠼⠑ ::t 5
+::s ⠼⠋ ::t 6
+::s ⠼⠛ ::t 7
+::s ⠼⠓ ::t 8
+::s ⠼⠊ ::t 9
+::s ⠼⠚ ::t 0
+
+::s ⠂ ::t ,
+::s ⠆ ::t ;
+::s ⠒ ::t : 
+::s ⠲ ::t .
+::s ⠦ ::t ?
+::s ⠖ ::t !
+::s ⠄ ::t '
+::s ⠤ ::t -
+::s ⠨⠤ ::t _
+
+::s ⠀ ::t " " ::comment blank
+# ::s ⠐ t " " ::comment blank in numeric mode
+::s ⠈ ::t "" ::comment accent
+# ::s ⠌ ::t / ::comment in numeric mode only
+# ::s ⠐ ::comment abbreviation sign
+# ::s ⠘ ::comment abbreviation sign
+# ::s ⠠ ::comment capital indicator
+::s ⠨ ::t . ::comment decimal point; emphasis
+::s ⠰ ::t "" ::comment letter indicator
+# ::s ⠴ ::t ”
+# ::s ⠶ ::t ()
+# ::s ⠸ ::comment abbreviation sign
+::s ⠼ ::t "" ::comment number indicator
+::s ⠘⠚ ::t ° ::word-external-punctuation
+::s ⠘⠚⠠⠉ ::t °C
+::s ⠘⠚⠉ ::t °C
+::s ⠘⠚⠠⠋ ::t °F
+::s ⠘⠚⠋ ::t °F
+
+::s ⠠⠶ ::t " ::word-external-punctuation
+::s ⠘⠦ ::t “ ::word-external-punctuation
+::s ⠘⠴ ::t ” ::word-external-punctuation
+::s ⠄⠦ ::t ‘
+::s ⠄⠴ ::t ’
+::s ⠠⠴ ::t ���
+::s ⠐⠣ ::t ( ::word-external-punctuation
+::s ⠐⠜ ::t ) ::word-external-punctuation
+::s ⠨⠣ ::t [ ::word-external-punctuation
+::s ⠨⠜ ::t ] ::word-external-punctuation
+::s ⠸⠣ ::t { ::word-external-punctuation
+::s ⠸⠜ ::t } ::word-external-punctuation
+::s ⠈⠣ ::t < ::word-external-punctuation
+::s ⠈⠜ ::t > ::word-external-punctuation
+::s ⠸⠌ ::t / ::word-external-punctuation
+::s ⠸⠡ ::t \ ::word-external-punctuation
+::s ⠠⠤ ::t – ::word-external-punctuation
+::s ⠐⠠⠤ ::t — ::word-external-punctuation
+::s ⠈⠯ ::t & ::word-external-punctuation
+::s ⠐⠔ ::t * ::word-external-punctuation
+::s ⠨⠦ ::t ∩ ::word-external-punctuation
+::s ⠨⠴ ::t % ::word-external-punctuation
+::s ⠐⠖ ::t + ::word-external-punctuation
+::s ⠐⠤ ::t − ::word-external-punctuation
+::s ⠐⠶ ::t = ::word-external-punctuation
+::s ⠈⠎ ::t $ ::word-external-punctuation
+::s ⠈⠉ ::t ¢ ::word-external-punctuation
+::s ⠈⠇ ::t £ ::word-external-punctuation
+::s ⠈⠽ ::t ¥ ::word-external-punctuation
+::s ⠈⠁ ::t @ ::word-external-punctuation
+::s ⠸⠹ ::t # ::word-external-punctuation
+::s ⠸⠲ ::t • ::word-external-punctuation
+::s ⠈⠢ ::t ^ ::word-external-punctuation
+::s ⠈⠔ ::t ~ ::word-external-punctuation
+::s ⠘⠉ ::t © ::word-external-punctuation
+::s ⠐⠌ ::t ÷ ::word-external-punctuation
+::s ⠐⠦ ::t × ::word-external-punctuation
+::s ⠈⠠⠹ ::t † ::word-external-punctuation
+::s ⠈⠠⠻ ::t ‡ ::word-external-punctuation
+::s ⠘⠏ ::t ¶ ::word-external-punctuation
+::s ⠘⠎ ::t § ::word-external-punctuation
+::s ⠘⠗ ::t ® ::word-external-punctuation
+::s ⠘⠞ ::t ™ ::word-external-punctuation
+
+# English Braille
+::s ⠁⠃ ::t about ::lcode eng ::use-only-for-whole-word
+::s ⠁⠃⠧ ::t above ::lcode eng ::use-only-for-whole-word
+::s ⠁⠉ ::t according ::lcode eng ::use-only-for-whole-word
+::s ⠁⠉⠗ ::t across ::lcode eng ::use-only-for-whole-word
+::s ⠁⠋ ::t after ::lcode eng ::use-only-for-whole-word
+::s ⠁⠋⠝ ::t afternoon ::lcode eng ::use-only-for-whole-word
+::s ⠁⠋⠺ ::t afterward ::lcode eng ::use-only-for-whole-word
+::s ⠁⠛ ::t again ::lcode eng ::use-only-for-whole-word
+::s ⠁⠛⠌ ::t against ::lcode eng ::use-only-for-whole-word
+::s ⠠⠽ ::t ally ::lcode eng ::use-only-at-end-of-word ::use-only-in-lower-case-environment
+::s ⠁⠇⠍ ::t almost ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠗ ::t already ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇ ::t also ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠹ ::t although ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠞ ::t altogether ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠺ ::t always ::lcode eng ::use-only-for-whole-word
+::s ⠨⠑ ::t ance ::lcode eng
+::s ⠯ ::t and ::lcode eng
+::s ⠜ ::t ar ::lcode eng
+::s ⠵ ::t as ::lcode eng ::use-only-for-whole-word
+::s ⠠⠝ ::t ation ::lcode eng ::use-only-at-end-of-word ::use-only-in-lower-case-environment
+::s ⠃ ::t b ::lcode eng
+::s ⠆ ::t bb ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠆ ::t be ::lcode eng ::use-only-at-start-of-word
+::s ⠆⠉ ::t because ::lcode eng ::use-only-for-whole-word
+::s ⠆⠋ ::t before ::lcode eng ::use-only-for-whole-word
+::s ⠆⠓ ::t behind ::lcode eng ::use-only-for-whole-word
+::s ⠆⠇ ::t below ::lcode eng ::use-only-for-whole-word
+::s ⠆⠝ ::t beneath ::lcode eng ::use-only-for-whole-word
+::s ⠆⠎ ::t beside ::lcode eng ::use-only-for-whole-word
+::s ⠆⠞ ::t between ::lcode eng ::use-only-for-whole-word
+::s ⠆⠽ ::t beyond ::lcode eng ::use-only-for-whole-word
+::s ⠃⠇ ::t blind ::lcode eng ::use-only-for-whole-word
+::s ⠃⠗⠇ ::t Braille ::lcode eng ::use-only-for-whole-word
+::s ⠃ ::t but ::lcode eng ::use-only-for-whole-word
+::s ⠉ ::t c ::lcode eng
+::s ⠉ ::t can ::lcode eng ::use-only-for-whole-word
+::s ⠸⠉ ::t cannot ::lcode eng
+::s ⠒ ::t cc ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠉⠧ ::t ceive ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠧⠙ ::t ceived ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠧⠎ ::t ceives ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠧⠛ ::t ceiving ::lcode eng
+::s ⠡ ::t ch ::lcode eng
+::s ⠐⠡ ::t character ::lcode eng
+::s ⠡ ::t child ::lcode eng ::use-only-for-whole-word
+::s ⠡⠝ ::t children ::lcode eng ::use-only-for-whole-word
+::s ⠒ ::t con ::lcode eng ::use-only-at-start-of-word
+::s ⠒ ::t : ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠙ ::t could ::lcode eng ::use-only-for-whole-word
+::s ⠙ ::t d ::lcode eng
+::s ⠙ ::t do ::lcode eng ::use-only-for-whole-word
+::s ⠐⠙ ::t day ::lcode eng
+# ::s ⠲ ::t dd ::t-alt . ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word ::comment abolished; interferes with period in abbrevisations such as U.S.
+::s ⠙⠉⠇ ::t declare ::lcode eng
+::s ⠙⠉⠇⠛ ::t declaring ::lcode eng
+::s ⠲ ::t dis ::lcode eng ::use-only-at-start-of-word
+::s ⠲ ::t .  ::lcode eng ::dont-use-at-start-of-word
+::s ⠑ ::t e ::lcode eng
+::s ⠂ ::t ea ::lcode eng ::dont-use-at-end-of-word
+::s ⠂ ::t , ::lcode eng ::use-only-at-end-of-word
+::s ⠫ ::t ed ::lcode eng
+::s ⠑⠊ ::t either ::lcode eng ::use-only-for-whole-word
+::s ⠢ ::t en ::lcode eng
+::s ⠰⠑ ::t ence ::lcode eng ::dont-use-at-start-of-word
+::s ⠢ ::t enough ::lcode eng ::use-only-for-whole-word
+::s ⠻ ::t er ::lcode eng
+::s ⠐⠑ ::t ever ::lcode eng
+::s ⠑ ::t every ::lcode eng ::use-only-for-whole-word
+::s ⠋ ::t f ::lcode eng
+::s ⠐⠋ ::t father ::lcode eng
+::s ⠖ ::t ff ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠋⠌ ::t first ::lcode eng
+::s ⠿ ::t for ::lcode eng
+::s ⠋⠗ ::t friend ::lcode eng ::use-only-for-whole-word
+::s ⠋⠗⠎ ::t friends ::lcode eng ::use-only-for-whole-word
+::s ⠋ ::t from ::lcode eng ::use-only-for-whole-word
+::s ⠰⠇ ::t ful ::lcode eng ::dont-use-at-start-of-word
+::s ⠛ ::t g ::lcode eng
+::s ⠶ ::t gg ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠣ ::t gh ::lcode eng
+::s ⠛ ::t go ::lcode eng ::use-only-for-whole-word
+::s ⠛⠙ ::t good ::lcode eng ::use-only-at-start-of-word
+::s ⠛⠗⠞ ::t great ::lcode eng
+::s ⠓ ::t h ::lcode eng
+::s ⠸⠓ ::t had ::lcode eng
+::s ⠓ ::t have ::lcode eng ::use-only-for-whole-word
+::s ⠐⠓ ::t here ::lcode eng
+::s ⠓⠻⠋ ::t herself ::lcode eng ::use-only-for-whole-word
+::s ⠓⠍ ::t him ::lcode eng ::use-only-for-whole-word
+::s ⠓⠍⠋ ::t himself ::lcode eng ::use-only-for-whole-word
+::s ⠦ ::t ? ::lcode eng
+::s ⠦ ::t his ::lcode eng ::use-only-for-whole-word
+::s ⠊⠍⠍ ::t immediate ::lcode eng ::use-only-for-whole-word
+::s ⠊⠍⠍⠇⠽ ::t immediately ::lcode eng ::use-only-for-whole-word
+::s ⠔ ::t in ::lcode eng
+::s ⠔⠒ ::t incon ::lcode eng ::use-only-at-start-of-word
+::s ⠬ ::t ing ::lcode eng
+::s ⠭ ::t it ::lcode eng ::use-only-for-whole-word
+::s ⠭⠎ ::t its ::lcode eng ::use-only-for-whole-word
+::s ⠭⠋ ::t itself ::lcode eng ::use-only-for-whole-word
+::s ⠰⠽ ::t ity ::lcode eng ::dont-use-at-start-of-word
+::s ⠚ ::t j ::lcode eng
+::s ⠚ ::t just ::lcode eng ::use-only-for-whole-word
+::s ⠅ ::t k ::lcode eng
+::s ⠐⠅ ::t know ::lcode eng
+::s ⠅ ::t knowledge ::lcode eng ::use-only-for-whole-word
+::s ⠇ ::t l ::lcode eng
+::s ⠨⠎ ::t less ::lcode eng ::dont-use-at-start-of-word
+::s ⠇⠗ ::t letter ::lcode eng ::use-only-for-whole-word
+::s ⠇⠗⠎ ::t letters ::lcode eng ::use-only-for-whole-word
+::s ⠇ ::t like ::lcode eng ::use-only-for-whole-word
+::s ⠇⠇ ::t little ::lcode eng ::use-only-for-whole-word
+::s ⠐⠇ ::t lord ::lcode eng
+::s ⠍ ::t m ::lcode eng
+::s ⠸⠍ ::t many ::lcode eng
+::s ⠰⠞ ::t ment ::lcode eng ::dont-use-at-start-of-word
+::s ⠍ ::t more ::lcode eng ::use-only-for-whole-word
+::s ⠐⠍ ::t mother ::lcode eng
+::s ⠍⠡ ::t much ::lcode eng ::use-only-for-whole-word
+::s ⠍⠌ ::t must ::lcode eng ::use-only-for-whole-word
+::s ⠍⠽⠋ ::t myself ::lcode eng ::use-only-for-whole-word
+::s ⠝  ::t n ::lcode eng
+::s ⠐⠝ ::t name ::lcode eng
+::s ⠝⠑⠉ ::t necessary ::lcode eng ::use-only-for-whole-word
+::s ⠝⠑⠊ ::t neither ::lcode eng ::use-only-for-whole-word
+::s ⠰⠎ ::t ness ::lcode eng ::dont-use-at-start-of-word
+::s ⠝  ::t not ::lcode eng ::use-only-for-whole-word
+::s ⠕⠄⠉ ::t o'clock ::lcode eng ::use-only-for-whole-word
+::s ⠷ ::t of ::lcode eng
+::s ⠐⠕ ::t one ::lcode eng
+::s ⠰⠛ ::t ong ::lcode eng ::dont-use-at-start-of-word
+::s ⠳ ::t ou ::lcode eng
+::s ⠨⠙ ::t ound ::lcode eng
+::s ⠨⠞ ::t ount ::lcode eng
+::s ⠐⠳ ::t ought ::lcode eng
+::s ⠳⠗⠧⠎ ::t ourselves ::lcode eng ::use-only-for-whole-word
+::s ⠳ ::t out ::lcode eng ::use-only-for-whole-word
+::s ⠪ ::t ow ::lcode eng
+::s ⠏ ::t p ::lcode eng
+::s ⠏⠙ ::t paid ::lcode eng ::use-only-for-whole-word
+::s ⠐⠏ ::t part ::lcode eng
+::s ⠏ ::t people ::lcode eng ::use-only-for-whole-word
+::s ⠏⠻⠓ ::t perhaps ::lcode eng ::use-only-for-whole-word
+::s ⠟ ::t q ::lcode eng
+::s ⠐⠟ ::t question ::lcode eng
+::s ⠟⠅ ::t quick ::lcode eng ::use-only-for-whole-word
+::s ⠟⠅⠻ ::t quicker ::lcode eng ::use-only-for-whole-word
+::s ⠟⠅⠑⠌ ::t quickest ::lcode eng ::use-only-for-whole-word
+::s ⠟ ::t quite ::lcode eng ::use-only-for-whole-word
+::s ⠗ ::t r ::lcode eng
+::s ⠗ ::t rather ::lcode eng ::use-only-for-whole-word
+::s ⠐⠗ ::t right ::lcode eng
+::s ⠗⠚⠉ ::t rejoice ::lcode eng
+::s ⠗⠚⠉⠛ ::t rejoicing ::lcode eng
+::s ⠎ ::t s ::lcode eng
+::s ⠎⠙ ::t said ::lcode eng ::use-only-for-whole-word
+::s ⠩ ::t sh ::lcode eng
+::s ⠩ ::t shall ::lcode eng ::use-only-for-whole-word
+::s ⠩⠙ ::t should ::lcode eng ::use-only-for-whole-word
+::s ⠨⠝ ::t sion ::lcode eng
+::s ⠎ ::t so ::lcode eng ::use-only-for-whole-word
+::s ⠐⠎ ::t some ::lcode eng
+::s ⠸⠎ ::t spirit ::lcode eng
+::s ⠌ ::t st ::lcode eng
+::s ⠌ ::t still ::lcode eng ::use-only-for-whole-word
+::s ⠎⠡ ::t such ::lcode eng ::use-only-for-whole-word
+::s ⠞ ::t t ::lcode eng
+::s ⠹ ::t th ::lcode eng
+::s ⠞ ::t that ::lcode eng ::use-only-for-whole-word
+::s ⠹ ::t this ::lcode eng ::use-only-for-whole-word
+::s ⠮ ::t the ::lcode eng
+::s ⠸⠮ ::t their ::lcode eng
+::s ⠮⠍⠧⠎ ::t themselves ::lcode eng ::use-only-for-whole-word
+::s ⠐⠮ ::t there ::lcode eng
+::s ⠘⠮ ::t these ::lcode eng
+::s ⠘⠹ ::t those ::lcode eng
+::s ⠐⠹ ::t through ::lcode eng
+::s ⠐⠞ ::t time ::lcode eng
+::s ⠰⠝ ::t tion ::lcode eng ::dont-use-at-start-of-word
+::s ⠖ ::t to ::lcode eng ::use-only-for-whole-word
+::s ⠞⠙ ::t today ::lcode eng ::use-only-for-whole-word
+::s ⠞⠛⠗ ::t together ::lcode eng ::use-only-for-whole-word
+::s ⠞⠍ ::t tomorrow ::lcode eng ::use-only-for-whole-word
+::s ⠞⠝ ::t tonight ::lcode eng ::use-only-for-whole-word
+::s ⠥ ::t u ::lcode eng
+::s ⠥⠝⠒ ::t uncon ::lcode eng ::use-only-at-start-of-word
+::s ⠥ ::t us ::lcode eng ::use-only-for-whole-word
+::s ⠠⠥⠲⠎⠲ ::t U.S. ::lcode eng
+::s ⠐⠥ ::t under ::lcode eng
+::s ⠘⠥ ::t upon ::lcode eng
+::s ⠧ ::t v ::lcode eng
+::s ⠧ ::t very ::lcode eng ::use-only-for-whole-word
+::s ⠺ ::t w ::lcode eng
+::s ⠴ ::t " ::lcode eng
+::s ⠴ ::t was ::lcode eng ::use-only-for-whole-word
+::s ⠶ ::t were ::lcode eng ::use-only-for-whole-word
+::s ⠱ ::t wh ::lcode eng
+::s ⠐⠱ ::t where ::lcode eng
+::s ⠱ ::t which ::lcode eng ::use-only-for-whole-word
+::s ⠘⠱ ::t whose ::lcode eng
+::s ⠺ ::t will ::lcode eng ::use-only-for-whole-word
+::s ⠾ ::t with ::lcode eng
+::s ⠘⠺ ::t word ::lcode eng
+::s ⠐⠺ ::t work ::lcode eng
+::s ⠸⠺ ::t world ::lcode eng
+::s ⠺⠙ ::t would ::lcode eng ::use-only-for-whole-word
+::s ⠭ ::t x ::lcode eng
+::s ⠽ ::t y ::lcode eng
+::s ⠽ ::t you ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗ ::t your ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗⠎ ::t yours ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗⠋ ::t yourself ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗⠧⠎ ::t yourselves ::lcode eng ::use-only-for-whole-word
+::s ⠐⠽ ::t young ::lcode eng
+::s ⠵ ::t z ::lcode eng
+::s ⠠⠴ ::t ’ ::lcode eng
+
+::preserve ::from U+2190 ::to U+21FF ::comments Arrows
+::preserve ::from U+2200 ::to U+22FF ::comment Mathematical Operators
+::preserve ::from U+2300 ::to U+23FF ::comment Miscellaneous Technical
+::preserve ::from U+2500 ::to U+257F ::comment Box Drawing
+::preserve ::from U+2580 ::to U+259F ::comment Block Elements
+::preserve ::from U+25A0 ::to U+25FF ::comment Geometric Shapes
+::preserve ::from U+2600 ::to U+26FF ::comment Miscellaneous Symbols
+::preserve ::from U+27C0 ::to U+27ED ::comment Miscellaneous Mathematical Symbols-A
+::preserve ::from U+27F0 ::to U+27FF ::comment Supplemental Arrows-A
+::preserve ::from U+2900 ::to U+297F ::comment Supplemental Arrows-B
+::preserve ::from U+2980 ::to U+29FF ::comment Miscellaneous Mathematical Symbols-B
+::preserve ::from U+2A00 ::to U+2AFF ::comment Supplemental Mathematical Operators
+::preserve ::from U+2B00 ::to U+2BFF ::comment Miscellaneous Symbols and Arrows
+::preserve ::from U+2E00 ::to U+2E27 ::comment Supplemental Punctuation (excluding ⸨⸩)
+::preserve ::from U+2E2A ::to U+2E7F ::comment Supplemental Punctuation (cont'd)
+::preserve ::from U+18B00 ::to U+18CD5 ::comment Khitan Small Script
+::preserve ::from U+1D100 ::to U+1D1FF ::comment Musical Symbols
+::preserve ::from U+1D6A8 ::to U+1D7CB ::comment Mathematical Alphanumeric Symbols (Greek)
+::preserve ::from U+1D800 ::to U+1DAAF ::comment Sutton SignWriting
+::preserve ::from U+1F800 ::to U+1F8FF ::comment Supplemental Arrows-C
+::preserve ::from U+1FA00 ::to U+1FA6F ::comment Chess Symbols
+::preserve ::from U+1FB00 ::to U+1FBCF ::comment Symbols for Legacy Computing
+::preserve ::from U+1FA70 ::to U+1FAFF ::comment Symbols and Pictographs Extended-A

uroman/data/romanization-table.v1.2.1.txt

@@ -0,0 +1,814 @@
+
+## European Latin extensions
+# Vowels
+::s Ä ::t Ae
+::s Ö ::t Oe
+::s Ü ::t Ue
+::s Å ::t Aa
+::s Æ ::t Ae
+::s Ø ::t oe
+::s Π::t Oe
+::s ä ::t ae
+::s ö ::t oe
+::s ü ::t ue
+::s å ::t aa
+::s æ ::t ae
+::s ø ::t oe
+::s œ ::t oe
+# Consonants
+::s Ç ::t S
+::s ç ::t s
+::s Ç ::t Ch ::lcode tur
+::s ç ::t ch ::lcode tur
+::s Ş ::t Sh
+::s ş ::t sh
+::s Ș ::t Sh
+::s ș ::t sh
+::s ß ::t ss
+::s Ț ::t Ts
+::s ț ::t ts
+
+# Miscellaneous
+::s ə ::t e
+
+# English
+::s chr ::t chr ::t-alt kr ::example chromosome, synchronize
+::s Chr ::t Chr ::t-alt Kr ::example Christmas, Chrysler
+::s eight ::t eight ::t-alt eit ::example eight, weight
+::s Eight ::t Eight ::t-alt Eit ::example Eighteen
+::s ight ::t ight ::t-alt ait ::example Knight
+::s gh ::t gh ::t-alt f, ph, "" ::example laugh, daughter
+::s high ::t high ::t-alt hai ::example highlight
+::s High ::t High ::t-alt Hai ::example High School
+::s Isle ::t Isle ::t-alt Ail ::use-only-at-start-of-word ::use-only-at-end-of-word ::example Isle
+::s Island ::t Island ::t-alt Ailand ::use-only-at-start-of-word ::use-only-at-end-of-word ::example Island
+::s kn ::t kn ::t-alt n ::use-only-at-start-of-word ::example knowledge
+::s Kn ::t Kn ::t-alt N ::use-only-at-start-of-word ::example Knight
+::s Mc ::t Mc ::t-alt Mac ::use-only-at-start-of-word ::example McNulty
+::s mc ::t mc ::t-alt mac ::use-only-at-start-of-word
+::s oo ::t oo ::t-alt u ::lcode eng ::example Brooklyn; Goose Bay
+::s ph ::t ph ::t-alt f ::example alpha
+::s Ph ::t Ph ::t-alt F ::example Philip
+::s Thom ::t Thom ::t-alt Tom ::use-only-at-start-of-word ::example Thomas, Thompson
+::s tion ::t tion ::t-alt shen ::example
+::s Sean ::t Sean ::t-alt Shawn ::use-only-at-start-of-word ::use-only-at-end-of-word
+::s ssion ::t ssion ::t-alt shen ::example Sessions
+::s St  ::t St ::t-alt Saint ::use-only-at-start-of-word ::use-only-at-end-of-word
+::s St.  ::t St. ::t-alt Saint ::use-only-at-start-of-word ::use-only-at-end-of-word
+::s Wr ::t Wr ::t-alt R ::example Wren
+::s wr ::t wr ::t-alt r ::example Cartwright
+::s x ::t x ::t-alt ks ::example Mexico
+::s x ::t x ::t-alt gz ::example example, anxiety, exhaust, exit
+
+# French
+::s â ::t a ::t-alt as ::example pâte/paste, pastry
+::s ê ::t e ::t-alt es ::example fête/feast
+::s î ::t i ::t-alt is ::example île/isle
+::s ô ::t o ::t-alt os ::example côte/coast
+::s û ::t u ::t-alt us ::example août/August
+::s eaux ::t eaux ::t-alt o ::example Bordeaux
+::s eau ::t eau ::t-alt o ::example Chateau
+::s auld ::t auld ::t-alt o ::use-only-at-end-of-word ::example Renauld
+::s ault ::t ault ::t-alt o ::use-only-at-end-of-word ::example Renault
+::s oux ::t oux ::t-alt u
+::s ois ::t ois ::t-alt oa ::use-only-at-end-of-word ::example Dubois
+
+# German
+::s Sch ::t Sch ::t-alt Sh
+::s sch ::t sch ::t-alt sh
+::s stein ::t stein ::t-alt shtain
+::s dt ::t dt ::t-alt tt ::use-only-at-end-of-word ::example Schmidt
+
+# Dutch
+::s ij ::t ij ::t-alt ai
+::s Ij ::t Ij ::t-alt Ai
+
+# Greek
+::s Ι ::t I
+::s ι ::t i
+::s ί ::t i
+::s ἶ ::t i
+::s Υ ::t Y
+::s υ ::t y
+::s Ρ ::t R
+::s ρ ::t r
+::s Ντ ::t D
+::s ντ ::t nd ::t-alt d
+# ::s ντζ ::t ntz
+::s Μπ ::t B
+::s μπ ::t mb ::t-alt b
+::s γγ ::t ng
+::s γκ ::t ng ::t-alt g
+::s ει ::t ei ::t-alt i
+::s ου ::t ou ::t-alt u
+::s χ ::t ch ::t-alt kh
+
+# Cyrillic
+::s Г ::t G ::t-alt H
+::s г ::t g ::t-alt h
+::s Е ::t E ::t-alt Ye
+::s е ::t e ::t-alt ye
+::s Ё ::t E ::t-alt Yo
+::s ё ::t e ::t-alt yo
+::s Х ::t Kh ::t-alt Ch, H ::comment Cyrillic capital ha
+::s х ::t kh ::t-alt ch, h ::comment Cyrillic small ha
+::s Щ ::t Shch ::t-alt Sh
+::s щ ::t shch ::t-alt sh
+::s Ъ ::t ::comment Cyrillic capital hard sign
+::s ъ ::t ::comment Cyrillic small hard sign
+::s Ы ::t Y ::comment Cyrillic capital yeru
+::s ы ::t y ::comment Cyrillic small yeru
+::s Ь ::t ::comment Cyrillic capital soft sign
+::s ь ::t ::comment Cyrillic small soft sign
+
+::s Ҥ ::t Ng ::comment Cyrillic capital ligature EN GHE
+::s ҥ ::t ng ::comment Cyrillic small ligature EN GHE
+::s Ә ::t e ::comment Cyrillic capital schwa
+::s ә ::t e ::comment Cyrillic small schwa
+::s Ӏ ::t ' ::comment Cyrillic palochka
+::s Ҵ ::t TS ::comment Cyrillic capital ligature te tse, used in Abkhasian
+::s ҵ ::t ts ::comment Cyrillic small ligature te tse, used in Abkhasian
+::s Ӕ ::t AE ::comment Cyrillic capital ligature a ie
+::s ӕ ::t ae ::comment Cyrillic small ligature a ie
+::s Г ::t H ::lcode ukr ::comment Ukrainian capital letter he 
+::s г ::t h ::lcode ukr ::comment Ukrainian small letter he
+::s Ґ ::t G ::lcode ukr ::comment Ukrainian capital letter ghe
+::s ґ ::t g ::lcode ukr ::comment Ukrainian small letter ghe
+
+# Gothic
+::s 𐌴 ::t e ::comment Gothic letter aihvus
+::s 𐌹 ::t i ::comment Gothic letter eis
+::s 𐍇 ::t x ::comment Gothic letter iggws
+
+# Georgian
+::s ა ::t a ::comment Georgian letter an
+::s ე ::t e ::comment Georgian letter en
+::s ი ::t i ::comment Georgian letter in
+::s ო ::t o ::comment Georgian letter on
+::s უ ::t u ::comment Georgian letter un
+
+# Armenian
+::s Ա ::t a ::comment Armenian capital letter ayb
+::s ա ::t a ::comment Armenian small letter ayb
+::s Ե ::t e ::comment Armenian capital letter ech
+::s ե ::t e ::comment Armenian small letter ech
+::s և ::t ev ::comment Armenian small ligature ech yiwn
+::s Է ::t e ::comment Armenian capital letter eh
+::s է ::t e ::comment Armenian small letter eh
+::s Ի ::t i ::comment Armenian capital letter ini
+::s ի ::t i ::comment Armenian small letter ini
+::s Օ ::t o ::comment Armenian capital letter oh
+::s օ ::t o ::comment Armenian small letter oh
+
+## Japanese
+# Katakana
+::s シ ::t shi
+::s チ ::t chi
+::s フ ::t fu
+::s ジ ::t ji
+::s ヂ ::t ji
+::s ヅ ::t zu
+::s シャ ::t sha
+::s シュ ::t shu
+::s ショ ::t sho
+::s チャ ::t cha
+::s チェ ::t che
+::s チュ ::t chu
+::s チョ ::t cho
+::s ジャ ::t ja
+::s ジュ ::t ju
+::s ジョ ::t jo
+::s ジェ ::t je
+::s ヂャ ::t ja
+::s ヂュ ::t ju
+::s ヂョ ::t jo
+::s フェ ::t fe
+::s ヴェ ::t ve
+::s フィ ::t fi
+::s ウィ ::t wi
+::s ヴィ ::t vi
+::s ティ ::t ti
+::s ディ ::t di
+::s ッ ::t (__SOKUON__) ::comment katakana double following consonant
+::s ー ::t (__CHOONPU__) ::comment katakana prolonged sound mark
+# Hiragana
+::s し ::t shi
+::s ち ::t chi
+::s つ ::t tsu
+::s ふ ::t fu
+::s を ::t o
+::s じ ::t ji
+::s ぢ ::t ji
+::s づ ::t zu
+::s しゃ ::t sha
+::s しゅ ::t shu
+::s しょ ::t sho
+::s ちゃ ::t cha
+::s ちゅ ::t chu
+::s ちょ ::t cho
+::s じゃ ::t ja
+::s じゅ ::t ju
+::s じょ ::t jo
+::s ぢゃ ::t ja
+::s ぢゅ ::t ju
+::s ぢょ ::t jo
+::s っ ::t (__SOKUON__) ::comment hiragana double following consonant
+::s 々 ::t ² ::comment ideographic iteration mark ::annotation repetition-sign
+
+::s フ ::t fu ::t-alt f
+::s キ ::t ki ::t-alt k
+::s ク ::t ku ::t-alt k
+::s ラ ::t ra ::t-alt la
+::s リ ::t ri ::t-alt li
+::s ル ::t ru ::t-alt lu, l, r
+::s レ ::t re ::t-alt le
+::s ロ ::t ro ::t-alt lo
+::s ム ::t mu ::t-alt m ::example キム = Kim
+::s シ ::t shi ::t-alt si ::example メキシコ = meksiko (Mexico)
+::s ス ::t su ::t-alt s
+::s ト ::t to ::t-alt t
+::s ツ ::t tsu ::t-alt tu, ts ::example シュルツ = Schultz
+
+# Chinese
+::s 邦 ::t bang ::t-alt bon, bum, bun, pon
+::s 鲍 ::t bao ::t-alt bow
+::s 堡 ::t bao ::t-alt berg, burg, bourg, burgh
+::s 贝 ::t bei ::t-alt ber
+::s 本 ::t ben ::t-alt bern, bon, bourn, burn
+::s 彼得 ::t bide ::t-alt peter, pet
+::s 伯 ::t bo ::t-alt ber
+::s 波 ::t bo ::t-alt po
+::s 布 ::t bu ::t-alt b
+::s 策 ::t ce ::t-alt tze, tzer
+::s 曾 ::t ceng ::t-alt tzen, zen
+::s 彻 ::t che ::t-alt tche
+::s 茨 ::t ci ::t-alt ts, tz, z
+::s 兹 ::t ci ::t-alt ds, dz, tz, z, zi
+::s 蒂 ::t di ::t-alt ti, tti
+::s 丁 ::t ding ::t-alt din, tin
+::s 顿 ::t dun ::t-alt ton
+::s 多 ::t duo ::t-alt do, dor, to
+::s 尔 ::t er ::t-alt l, le, ll, r
+::s 弗 ::t fu ::t-alt f, fer, pher, v, ver, vir
+::s 夫 ::t fu ::t-alt f, v, v
+::s 福 ::t fu ::t-alt faw, for, ford
+::s 哥 ::t ge ::t-alt go, co
+::s 戈 ::t ge ::t-alt go
+::s 各 ::t ge ::t-alt go, co
+::s 赫 ::t he ::t-alt ch, che, cher, ge
+::s 华 ::t hua ::t-alt ver, wa, war, wer ::example Washington
+::s 怀 ::t huai ::t-alt whi, wi, wy
+::s 惠 ::t hui ::t-alt wha, whea
+::s 基 ::t ji ::t-alt ki, chi
+::s 吉 ::t ji ::t-alt gi, gui
+::s 加 ::t jia ::t-alt ca, ga, ka ::example Canada
+::s 杰 ::t jie ::t-alt ger
+::s 金 ::t jin ::t-alt kin, gin
+::s 斤 ::t jin ::t-alt zin
+::s 康 ::t kang ::t-alt con, corn
+::s 考 ::t kao ::t-alt cow, cour
+::s 克 ::t ke ::t-alt k, che, cher
+::s 科 ::t ke ::t-alt ko
+::s 拉 ::t la ::t-alt ra ::example Tirana
+::s 朗 ::t lang ::t-alt lon, ron
+::s 赖 ::t lai ::t-alt ri
+::s 劳 ::t lao ::t-alt low
+::s 勒 ::t lei ::t-alt ler
+::s 伦 ::t lun ::t-alt lon, ran, ron
+::s 里 ::t li ::t-alt ri
+::s 利 ::t li ::t-alt ri ::example Ferrari
+::s 隆 ::t long ::t-alt lon, lum, lund
+::s 罗 ::t luo ::t-alt l, lo, lu, ro, row, ru
+::s 洛 ::t luo ::t-alt lo, low, ro
+::s 默 ::t mo ::t-alt mer
+::s 纳 ::t na ::t-alt ne, ner
+::s 珀 ::t po ::t-alt per
+::s 奇 ::t qi ::t-alt chi, dge, ge, tch
+::s 齐 ::t qi ::t-alt tsi, zi
+::s 乔 ::t qiao ::t-alt jo
+::s 青 ::t qing ::t-alt tsing
+::s 琼 ::t qiong ::t-alt jon, jum, jun
+::s 瑟 ::t se ::t-alt the
+::s 什 ::t shen ::t-alt sh
+::s 圣 ::t sheng ::t-alt san, sao, saint
+::s 斯 ::t si ::t-alt s, rth, th ::example Alaska
+::s 索 ::t suo ::t-alt tho
+::s 特 ::t te ::t-alt t
+::s 翁 ::t weng ::t-alt on
+::s 沃 ::t wo ::t-alt ver, vo, war, wer
+::s 乌 ::t wu ::t-alt ou, u
+::s 希 ::t xi ::t-alt chi, hi, shi
+::s 西 ::t xi ::t-alt s, si
+::s 锡 ::t xi ::t-alt ci, si, thi, zi
+::s 夏 ::t xia ::t-alt ha, cha, cia, sha, tia
+::s 香 ::t xiang ::t-alt chan, cham
+::s 歇 ::t xie ::t-alt she
+::s 谢 ::t xie ::t-alt che, she
+::s 辛 ::t xin ::t-alt cin, sen, sin, sing, sun, zen
+::s 欣 ::t xin ::t-alt hin, shin
+::s 休 ::t xiu ::t-alt hu, hue
+::s 修 ::t xiu ::t-alt ciu, siu, thew, tiu
+::s 许 ::t xu ::t-alt hue, schue
+::s 逊 ::t xun ::t-alt son
+::s 耶 ::t ye ::t-alt yer, ier
+::s 泽 ::t ze ::t-alt ser
+::s 扎 ::t zha ::t-alt za
+::s 詹 ::t zhan ::t-alt ja, jam, jan, jen, jon
+::s 治 ::t zhi ::t-alt ge ::example George
+
+## Numbers
+# Chinese and Japanese numbers
+::s 零 ::num 0
+::s 〇 ::num 0
+::s 一 ::num 1
+::s 二 ::num 2
+::s 三 ::num 3
+::s 四 ::num 4
+::s 五 ::num 5
+::s 六 ::num 6
+::s 七 ::num 7
+::s 八 ::num 8
+::s 九 ::num 9
+::s 十 ::num 10
+::s 百 ::num 100
+::s 千 ::num 1000
+::s 万 ::num 10000
+::s 萬 ::num 10000
+::s 亿 ::num 100000000
+::s 億 ::num 100000000
+::s 兆 ::num 1000000000000
+::s 京 ::num 10000000000000000
+
+::s 北京 ::t beijing
+::s 京都 ::t jingdou
+::s 东京 ::t dongjing
+::s 京胡 ::t jinghu
+::s 南京 ::t nangjing
+::s 普京 ::t pujing ::comment Putin
+::s 東京 ::t dongjing ::comment Tokyo
+::s 京兆 ::t jingzhao
+
+::s ㎢ ::t km²
+::s ㎥ ::t m³
+::s ㎝ ::t cm
+
+## Indian
+# see mostly under UnicodeDataOverwrite.txt
+
+# Malayalam
+::s ൗ ::t au ::comment MALAYALAM AU LENGTH MARK
+
+# Tamil
+::s ட ::t d ::comment most commonly d, but t when word-initial or in a doubled consonant
+::s ஃப ::t f ::comment h+p=f
+::s ஃஜ ::t z ::comment h+j=z
+
+# Myanmar/Burmese
+# ::s ့ ::t ::comment dot below, denotes creaky tone 
+# ::s း ::t ::comment visarga, denotes high tone
+::s ၌ ::t -nai ::comment locative
+::s ၍ ::t -jwe ::comment completed
+::s ၎ ::t legau ::comment aforementioned
+::s ၏ ::t -i ::comment genetive
+
+# Lao
+::s ັ ::t a ::comment vowel sign mai kan
+::s ົ ::t o ::comment vowel sign mai kon
+::s ູ ::t uu ::comment vowel sign uu
+::s ຽ ::t y ::comment semivowel sign nyo
+::s ຼ ::t l ::comment semivowel sign lo
+::s ລ ::t l ::comment lo loot
+::s ຣ ::t l ::comment lo ling
+::s ໝ ::t m ::comment ho mo
+::s ໜ ::n ::comment ho no
+::s ຢ ::t y ::comment yo
+::s ໍ ::t oo ::comment niggahita (possibly also nasal -m in final position)
+::s ໆ ::t ² ::comment Lao ko la ::annotation repetition-sign
+::s ຯ ::t ...  ::comment Lao ellipsis
+
+# Thai
+::s ออ ::t o
+::s อั ::t a
+::s อิ ::t i
+::s ๆ ::t ² ::comment Thai character maiyamok ::annotation repetition-sign
+
+# Khmer
+::s ័ ::t "" ::comment Khmer samyok sannya: indicates deviation from the general rules of pronunciation
+::s ៏ ::t "" ::comment Khmer sign ahsda: denotes stressed intonation in some single-consonant words
+::s ៍ ::t "" ::comment Khmer sign toandakhiat: indicates that the base character is not pronounced
+::s ៌ ::t "" ::comment Khmer sign robat: a diacritic historically corresponding to the repha form of ra in Devanagari
+::s ប៉ ::t pa ::comment Khmer ba + musĕkâtônd -> pa
+::s ៗ ::t ² ::comment Khmer sign lek too ::annotation repetition-sign
+
+## Semitic languages
+# Arabic
+::s و ::t w  ::comment Arabic letter waw  ::t-alt o, u  ::lcode ara
+::s ء ::t ' ::comment hamza 
+::s ٔ ::t ' ::comment hamza above
+::s ٕ ::t ' ::comment hamza below
+::s ع ::t ' ::comment ain
+::s آ ::t a ::comment alef madda
+::s ٓا ::t a ::comment Arabic maddah above plus alef (presumably an ill-formed version of آ; found 1 instance in Urdu text)
+::s إ ::t i ::comment alef with hamza below
+::s ٱ ::t a ::comment alef wasla ::comment typically indicates liaison with preceding word 
+::s ة ::t a ::comment teh marbuta
+::s ۃ ::t a ::comment teh marbuta goal ::comment Used in Punjabi, Sindhi. Different from plain 'teh marbuta'?
+::s ي ::t y ::comment Arabic yeh
+::s ى ::t a ::comment alef maksura
+::s ﻯ ::t a ::comment alef maksura isolated form
+::s ﻰ ::t a ::comment alef maksura final form
+::s ﯨ ::t a ::comment Uighur Kazach Kirghiz alef maksura initial form
+::s ﯩ ::t a ::comment Uighur Kazach Kirghiz alef maksura medial form
+::s ٰ ::t a  ::comment Arabic letter superscript alef
+::s ـ ::t   ::comment tatweel (filler)
+::s َ ::t a ::comment fatha ("-a")
+::s ُ ::t u ::comment damma ("-u")
+::s ِ ::t i ::comment kasra ("-i")
+::s ْ ::t   ::comment sukun (no vowel)
+::s ۡ ::t  ::comment small high dotless head of khah; like sukun (no vowel); used in Kashmiri, Assamese
+::s ً ::t   ::comment fathatan ("-an")
+::s اً ::t an ::comment alef + fathatan
+::s ٌ ::t   ::comment dammatan ("-un")
+::s ٍ ::t   ::comment kasratan ("-in")
+::s ّ ::t   ::comment shadda (consonant doubler)
+::s ڃ ::t ny ::comment Arabic letter nyeh U+0683 (used in Sindhi (snd))
+::s ڄ ::t dy ::comment Arabic letter dyeh U+0684 (used in Sindhi (snd))
+::s ۾ ::t men ::comment Sindhi postposition men
+::s ؑ ::t alayhe wasallam ::comment "upon him be peace"
+::s ﷴ ::t mohammad ::comment "Mohammad"
+::s ﷸ ::t wasallam ::comment "and peace"
+::s ﷺ ::t sallallahou alayhe wasallam ::comment "prayer of God be upon him and his family and peace"
+
+# Farsi
+::s ی ::t i ::t-alt y ::comment Contributed by Nima
+::s ای ::t i ::t-alt ai ::use-only-at-start-of-word ::comment Contributed by Nima
+::s هٔ ::t eye ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s و ::t v ::t-alt o, u ::lcode fas ::comment Arabic letter waw
+::s ض ::t z ::t-alt d ::lcode fas ::comment Contributed by Marjan
+::s ث ::t s ::t-alt th ::lcode fas ::comment Contributed by Marjan
+::s ذ ::t z ::t-alt th ::lcode fas ::comment Contributed by Nima
+::s ع ::t a ::t-alt ' ::lcode fas ::comment Contributed by Nima
+::s عا ::t a ::lcode fas ::comment Contributed by Nima
+::s عی ::t i ::t-alt iy ::lcode fas ::comment Contributed by Nima
+::s عو ::t u ::t-alt o, av ::lcode fas ::comment Contributed by Nima
+::s چ ::t ch ::t-alt tch, tsh ::lcode fas ::comment Contributed by Nima
+::s ه ::t e ::t-alt h ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s ‌ ::t "" ::t-alt " " ::lcode fas ::comment source is character "zero-width non-joiner" (U+200C); Contributed by Nima
+::s غ ::t gh ::t-alt g ::lcode fas
+::s آئی ::t ai ::t-alt ae ::lcode fas
+::s ائی ::t ai ::t-alt ae ::lcode fas
+::s آئو ::t au ::t-alt ao ::lcode fas
+::s ائو ::t au ::t-alt ao ::lcode fas
+
+# Kashmiri (so far: educated guesses)
+::s ٖ ::t a ::comment Arabic subscript alef U+0656
+::s ٗ ::t u ::comment Arabic inverted damma U+0657
+::s ۚ ::t j ::comment Arabic small high jeem U+06DA
+::s ۪ ::t   ::comment Arabic emtpy centre low stop U+06EA
+::s ۬ ::t   ::comment Arabic rounded high stop with filled center U+06EC 
+
+# Pashto
+::s ٙ ::t e
+
+# Hebrew
+::s ב ::t v ::comment Hebrew letter bet  ::t-alt b
+::s כ ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s ך ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s פ ::t f ::comment Hebrew letter pe  ::t-alt p
+::s ש ::t sh  ::comment Hebrew letter shin  ::t-alt s
+::s ו ::t v  ::comment Hebrew letter vav  ::t-alt o, u
+::s ח ::t ch ::comment Hebrew letter het  ::t-alt h ::use-alt-in-pointed
+::s ק ::t q ::t-alt k ::use-alt-in-pointed
+::s וֹ ::t o
+::s וּ ::t u
+::s קְוָ ::t qva ::t-alt kva  ::use-alt-in-pointed
+::s י ::t y
+::s יּ ::t y
+::s יָּ ::t ya
+::s ע ::t '
+::s ִי ::t i ::t-alt iy ::use-alt-in-pointed
+::s ֵי ::t e
+::s ִיּ ::t iy
+::s ִיָּ ::t iya
+::s ױ ::t oy
+::s א ::t a ::t-alt '
+::s אָ ::t a
+::s ֹא ::t o
+::s אַ ::t 'a
+::s אֲ ::t 'a
+::s אֶ ::t e
+::s אֱ ::t e
+::s פ ::t f
+::s פּ ::t p
+::s פַּ ::t pa
+::s פְּ ::t pe ::t-alt p ::use-alt-in-pointed 
+::s שׁ ::t sh
+::s שָׁ ::t sha
+::s שָּׁ ::t sha ::comment ?
+::s שְׁ ::t she ::t-alt sh ::use-alt-in-pointed
+::s שֶׁ ::t she
+::s שִׁ ::t shi
+::s שֻׁ ::t shu
+::s שׂ ::t s
+::s שָׂ ::t sa
+::s שְׂ ::t s ::t-alt se ::use-alt-in-pointed
+::s כּ ::t k
+::s כֶּ ::t ke
+::s כֹּ ::t ko
+::s בּ ::t b
+::s בַּ ::t ba
+::s בָּ ::t ba
+::s בְּ ::t be ::t-alt b ::use-alt-in-pointed
+::s בֶּ ::t be
+::s תּ ::t t
+::s תַּ ::t ta
+::s תֵּ ::t te
+::s תִּ ::t ti
+::s דָּ ::t da
+::s דְּ ::t de ::t-alt d ::use-alt-in-pointed
+::s גּ ::t g
+::s לֵּ ::t le
+::s ד׳ ::t dh
+::s ג׳ ::t j
+::s ת׳ ::t th
+::s ז׳ ::t zh
+::s חַ ::t ach ::comment furtive patah ::use-only-at-end-of-word
+::s עַ ::t a' ::comment furtive patah ::use-only-at-end-of-word
+::s הַּ ::t ah ::comment furtive patah ::use-only-at-end-of-word
+::s ַ ::t a ::comment Hebrew point patah
+::s ֲ ::t a ::comment Hebrew point hataf patah (hataf = reduced)
+::s ֳ ::t o ::comment Hebrew point hataf qamats
+::s ָ ::t a ::comment Hebrew point qamats ::t-alt o ::use-alt-in-pointed
+::s ֶ ::t e ::comment Hebrew point segol
+::s ֱ ::t e ::comment Hebrew point hataf segol (hataf = reduced)
+::s ְ ::t e ::comment Hebrew point sheva ::t-alt "" ::use-alt-in-pointed
+::s ֵ ::t e ::comment Hebrew point tsere
+::s ִ ::t i ::comment Hebrew point hiriq
+::s ֹ ::t o ::comment Hebrew point holam
+::s ֻ ::t u ::comment Hebrew point qubuts
+# ::s ּ ::t "" ::comment Hebrew point dagesh or mapiq
+
+# Yiddish
+::s א ::t a ::lcode yid ::comment called "silent" alef
+::s אי ::t y ::lcode yid
+::s איי ::t ey ::lcode yid
+::s או ::t u ::lcode yid
+::s אוי ::t oy ::lcode yid
+::s אַ ::t a ::lcode yid
+::s אָ ::t o ::lcode yid
+::s ב ::t b ::lcode yid
+::s בֿ ::t v ::lcode yid
+::s דזש ::t dzh ::lcode yid
+::s ו ::t u ::lcode yid
+::s וּ ::t u ::lcode yid
+::s וֹ ::t o ::lcode yid
+::s װ ::t v ::lcode yid
+::s ווא ::t wa ::lcode yid
+::s וואַ ::t wa ::lcode yid
+::s ווע ::t we ::lcode yid
+::s ווי ::t wi ::lcode yid
+::s וואוי ::t wo ::lcode yid
+::s וי ::t oy ::lcode yid
+::s זש ::t zh ::lcode yid
+::s ח ::t ch ::lcode yid
+::s טש ::t tsh ::lcode yid
+::s יִ::t i ::lcode yid
+::s יי ::t ey ::lcode yid ::comment maybe "yi" at beginning of word
+::s ײַ ::t ay ::lcode yid
+::s כּ ::t k ::lcode yid
+::s כ ::t ch ::lcode yid
+::s ך ::t ch ::lcode yid
+::s ע ::t e ::lcode yid
+::s פּ ::t p ::lcode yid
+::s פֿ ::t f ::lcode yid
+::s ף ::t f ::lcode yid ::comment sometimes p
+::s ק ::t k ::lcode yid
+::s ת ::t s ::lcode yid
+
+# Syriac/Aramaic (should be vetted by expert)
+::s ܰ ::t a  ::comment Syriac pthaha above
+::s ܲ ::t a  ::comment Syriac pthaha dotted
+::s ܳ ::t aa  ::comment Syriac zqapha above
+::s ܴ ::t aa  ::comment Syriac zqapha below
+::s ܵ ::t aa  ::comment Syriac zqapha dotted
+::s ܶ ::t e  ::comment Syriac rbasa above
+::s ܷ ::t e  ::comment Syriac rbasa below
+::s ܿ ::t o  ::comment Syriac rwaha
+::s ܸ ::t e  ::comment Syriac dotted zlama horizontal
+::s ܹ ::t e  ::comment Syriac dotted zlama angular
+::s ܺ ::t i  ::comment Syriac hbasa above
+::s ܝܺ ::t i  ::comment Syriac yudh + hbasa above
+::s ܼ ::t u  ::comment Syriac hbasa-esasa dotted
+::s ܽ ::t o  ::comment Syriac esasa above
+::s ܾ ::t u  ::comment Syriac esasa below
+::s ݇ ::t ""  ::comment Syriac oblique line above; indication of a silent letter
+
+::s ܖ ::t d  ::comment Syriac letter dotless dalath rish; ambiguous form for undifferentiated early dalath/rish
+::s ܜ ::t t  ::comment Syriac letter teth garshuni; used in Garshuni documents
+::s ܒ݂ ::t v  ::comment Syriac beth + rukkakha
+::s ܒ̥ ::t v  ::comment Syriac beth + ring-below
+::s ܓ݂ ::t g  ::comment Syriac gammal + rukkakha [IPA: ɣ]
+::s ܓ̥ ::t g  ::comment Syriac gammal + ring-below [IPA: ɣ]
+::s ܕ݂ ::t d  ::comment Syriac dalath + rukkakha [IPA: ð]
+::s ܕ̥ ::t d  ::comment Syriac dalath + ring-below [IPA: ð]
+::s ܟ݂ ::t kh  ::comment Syriac kaph + rukkakha [IPA: x]
+::s ܟ̥ ::t kh  ::comment Syriac kaph + ring-below [IPA: x]
+::s ܦ݂ ::t f  ::comment Syriac pe + rukkakha
+::s ܦ̥ ::t f  ::comment Syriac pe + ring-below
+::s ܦ݁ ::t p  ::comment Syriac pe + qushshaya
+::s ܬ݂ ::t th  ::comment Syriac taw + rukkakha [IPA: θ]
+::s ܬ̥ ::t th  ::comment Syriac taw + ring-below [IPA: θ]
+
+::s ܄ ::t : ::comment Syriac sublinear colon; used at the end of verses of supplicationscolon skewed left
+::s ܆ ::t , ::comment Syriac colon skewed left; marks a dependent clause
+::s ܇ ::t , ::comment Syriac colon skewed right; marks the end of a subdivision of the apodosis, or latter part of a Biblical verse
+
+# Uzbek
+::s ʻ ::t ' ::comment modifies pronunciation of preceding "o" and "g"
+::s ʼ ::t ' ::comment glottal stop (tutuq belgisi)
+
+# Uyghur
+::s ئا ::t a ::lcode uig
+::s ە ::t e ::lcode uig
+::s ئې ::t e ::lcode uig ::latinplus ë
+::s ې ::t e ::lcode uig ::latinplus ë
+::s ئە ::t e ::lcode uig
+::s يە ::t e ::lcode uig
+::s ئى ::t i ::lcode uig
+::s ى ::t i ::lcode uig
+::s ئو ::t o ::lcode uig
+::s و ::t o ::lcode uig
+::s ئۇ ::t u ::lcode uig
+::s ۇ ::t u ::lcode uig
+::s چ ::t ch ::t-alt q ::lcode uig
+::s خ ::t x ::lcode uig
+::s ژ ::t zh ::lcode uig
+::s ئۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ئۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۋ ::t w ::lcode uig
+
+# Maldivian
+::s ް ::t ::comment thaana sukun
+::s ަ ::t a ::comment thaana abafili
+::s ާ ::t aa ::comment thaana aabaafili
+::s ި ::t i ::comment thaana ibifili
+::s ީ ::t ee ::comment thaana eebeefili
+::s ު ::t u ::comment thaana ubufili
+::s ޫ ::t oo ::comment thaana ooboofili
+::s ެ ::t e ::comment thaana ebefili
+::s ޭ ::t ey ::comment thaana eybeyfili
+::s ޮ ::t o ::comment thaana obofili
+::s ޯ ::t oa ::comment thaana oaboafili
+
+# Canadian syllabics (Inuktitut)
+::s ᑊ ::t p ::comment syllable final
+::s ᐟ ::t t ::comment syllable final
+::s ᐠ ::t k ::comment syllable final
+::s ᐨ ::t c ::comment syllable final
+::s ᒼ ::t m ::comment syllable final
+::s ᐣ ::t n ::comment syllable final
+::s ᐢ ::t s ::comment syllable final
+::s ᐧ ::t y ::comment syllable final
+::s ᐤ ::t w ::comment syllable final
+::s ᐦ ::t h ::comment syllable final
+::s ᕽ ::t hk ::comment syllable final
+::s ᓫ ::t l ::comment syllable final
+::s ᕑ ::t r ::comment syllable final
+
+## Punctuation
+# delete
+::s ¿ ::t "" ::comment inverted question mark
+::s ¡ ::t "" ::comment inverted exclamation mark
+# preserve
+::s ′ ::t ′
+# Cyrillic
+::s ⁙ ::t . ::comment five dot punctuation
+# Amharic/Ethiopian
+::s ። ::t .
+::s ፣ ::t ,
+::s ፤ ::t ;
+::s ፥ ::t :
+::s ፡ ::t " " ::comment Ethiopic wordspace
+::s ፦ ::t :  ::comment Ethiopic preface colon
+::s ቸ ::t cha ::comment Ethiopic syllable ca
+::s ቹ ::t chu ::comment Ethiopic syllable cu
+::s ቺ ::t chi ::comment Ethiopic syllable ci
+::s ቻ ::t chaa ::comment Ethiopic syllable caa
+::s ቼ ::t chee ::comment Ethiopic syllable cee
+::s ች ::t che ::comment Ethiopic syllable ce
+::s ቾ ::t cho ::comment Ethiopic syllable co
+::s ሠ ::t sa ::comment Ethiopic syllable sza
+::s ሡ ::t su ::comment Ethiopic syllable szu
+::s ሢ ::t si ::comment Ethiopic syllable szi
+::s ሣ ::t saa ::comment Ethiopic syllable szaa
+::s ሤ ::t see::comment Ethiopic syllable szee
+::s ሥ ::t se ::comment Ethiopic syllable sze
+::s ሦ ::t so ::comment Ethiopic syllable szo
+::s ጠ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጡ ::t tu ::comment Ethiopic syllable thu with ejective 't'
+::s ጢ ::t ti ::comment Ethiopic syllable thi with ejective 't'
+::s ጣ ::t taa ::comment Ethiopic syllable thaa with ejective 't'
+::s ጤ ::t tee ::comment Ethiopic syllable thee with ejective 't'
+::s ጥ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጦ ::t to ::comment Ethiopic syllable tho with ejective 't'
+
+# Devanagari (Hindi etc.)
+::s । ::t . ::comment danda
+::s ॥ ::t . ::comment double danda
+::s ৷ ::t . ::comment Bengali currency numerator four; used as danda
+::s ॰ ::t . ::comment Devanagari abbreviation sign
+# Oriya/Odia (India)
+::s ୤ ::t . ::comment danda (deprecated, should use Devanagari danda ।)
+::s ୥ ::t . ::comment double danda (deprecated, should use Devanagari double danda ॥)
+# Tibetan
+::s ། ::t ,
+::s །: ::t :
+::s ༏ ::t ;
+::s ༎ ::t .
+::s ༑ ::t ,  ::comment Tibetan mark run chen spungs shad
+::s ༼ ::t (  ::comment Tibetan open roof punctuation
+::s ༽ ::t )  ::comment Tibetan close roof punctuation
+::s ༈ ::t ""  ::comment Tibetan mark srbul shad
+::s 【 ::t [  ::comment left black lenticular bracket
+::s 】 ::t ]  ::comment right black lenticular bracket
+::s ༄ ::t ""  ::comment Tibetan head mark
+::s ༄༅ ::t ""  ::comment Tibetan head mark
+::s ༆ ::t ""  ::comment Tibetan head mark
+# Myanmar/Burmese
+::s ၊ ::t ,
+::s ။ ::t .
+Khmer
+::s ៖ ::t ; ::comment Khmer sign camnuc pii kuuh
+::s ។ ::t . ::comment Khmer sign khan
+# Arabic
+::s ، ::t ,
+::s ؛ ::t ;
+::s ٬ ::t ,
+::s ۔ ::t .
+::s ؟ ::t ?
+::s ٪ ::t %
+::s ٫ ::t , ::comment Arabic decimal separator
+::s ۽ ::t & ::comment Arabic sign Sindhi ampersand
+# Aramaic
+::s ܀ ::t .
+::s ܂ ::t .
+# Hebrew
+::s ־ ::t - ::comment maqaf
+# Armenian
+::s ։ ::t .
+::s ՝ ::t , ::comment Armenian comma
+# Chinese
+::s ， ::t ", "
+::s 、 ::t ", "
+::s 。 ::t ". "
+::s ！ ::t "! "
+::s ？ ::t "? "
+::s 「 ::t ' "'
+::s 」 ::t '" '
+::s 《 ::t ' "'
+::s 》 ::t '" '
+::s （ ::t " ("
+::s ） ::t ") "
+::s ； ::t ;
+::s ： ::t ": "
+::s ︰ ::t ": "
+::s － ::t -
+::s ／ ::t /
+::s ＝ ::t =
+::s ～ ::t ~
+::s ＆ ::t &
+::s ＜ ::t <
+::s ＞ ::t >
+::s ％ ::t %
+::s 　 ::t " " ::comment ideographic space
+# Japanese
+::s 『 ::t ' "'
+::s 』 ::t '" '
+::s ・ ::t " "  ::comment Katakana middle dot; separates name elements such as first and last name
+
+# Symbols
+::s ∞ ::t ∞  ::comment infinity
+::s ­ ::t   ::comment soft hyphen; used to indicate preferred line breaks; remove
+::s ֊ ::t -  ::comment Armenian hyphen; map to regular hyphen-minus
+::s ᐩ ::t +  ::comment Canadian syllabics final plus; map to regular plus
+::s ﹐ ::t ,  ::comment small comma; map to regular comma
+::s ˚ ::t °  ::comment ring above; map to degree sign
+::s ⇒ ::t ⇒ ::comment rightwards double arrow
+::s † ::t †  ::comment dagger
+::s • ::t •  ::comment bullet
+::s ℃ ::t °C  ::comment degree Celsius; split into 2 characters
+::s ℉ ::t °F  ::comment degree Fahrenheit; split into 2 characters
+::s ― ::t ―  ::comment horizontal bar
+::s ˇ ::t ˇ  ::comment caron (sometimes apparently used for "Arabic vowel sign small v above" U+065A, e.g. in Gilaki language (glk))
+::s ″ ::t ″  ::comment double prime
+::s ﴾ ::t (  ::comment ornate left parenthesis
+::s ﴿ ::t )  ::comment ornate right parenthesis
+::s 〔 ::t [  ::comment left tortoise shell bracket
+::s 〕 ::t ]  ::comment right tortoise shell bracket
+::s ﹝ ::t (  ::comment small left tortoise shell bracket
+::s ﹞ ::t )  ::comment small left tortoise shell bracket
+::s ♄ ::t ♄  ::comment Saturn
+::s ♆ ::t ♆  ::comment Neptune
+::s ♋ ::t ♋  ::comment Cancer

uroman/data/string-distance-cost-rules.txt

@@ -0,0 +1,896 @@
+# String distance
+
+::s1 a ::s2 ::cost 0.1
+::s1 b ::s2 ::cost 1
+::s1 b ::s2 ::cost 0.2 ::left1 /[aou]m$/ ::right1 [e] ::lc1 eng ::lc2 zho ::example Balcombe
+::s1 c ::s2 ::cost 1
+::s1 c ::s2 ::cost 0.2 ::left1 /[aeou]$/ ::right1 [cgkq] ::lc2 zho
+::s1 c ::s2 ::cost 0.5 ::left1 /[aeou][lnr]?$/ ::right1 [h] ::lc2 zho
+::s1 d ::s2 ::cost 1
+::s1 d ::s2 ::cost 0.5 ::left1 /[aeiou][lnr]$/ ::right1 [-,$ ]
+::s1 d ::s2 ::cost 0.4 ::lc1 eng ::lc2 zho ::right1 [bcfgklmnpqrstvwxz]
+::s1 e ::s2 ::cost 0.1
+::s1 é ::s2 ::cost 0.1
+::s1 e ::s2 ::cost 0.02 ::lc2 fas
+::s1 e ::s2 ::cost 0.02 ::lc1 amh ::lc2 eng
+::s1 f ::s2 ::cost 1
+::s1 g ::s2 ::cost 1
+::s1 g ::s2 ::cost 0.4 ::right1 [bcdfghklmnpqrstvwxz] ::lc2 zho
+::s1 g ::s2 ::cost 0.2 ::right1 [k] ::lc2 zho
+::s1 h ::s2 ::cost 0.5
+::s1 h ::s2 ::cost 0.1 ::left1 /[aeiouy]$/ ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 h ::s2 ::cost 0.2 ::left1 /[bdlnr]$/ ::right1 [-,$ aeiouy] ::example Delhi, Minh, Riyadh
+::s1 i ::s2 ::cost 0.1
+::s1 j ::s2 ::cost 0.5
+::s1 k ::s2 ::cost 1
+::s1 l ::s2 ::cost 1
+::s1 l ::s2 ::cost 0.3 ::left1 /eui$/ ::right1 [-,$ ] ::example Argenteuil
+::s1 l ::s2 ::cost 0.3 ::left1 /a$/ ::right1 [km] ::comment walk, palm
+::s1 l ::s2 ::cost 0.3 ::left1 /[aeiou]$/ ::right1 [bdfgkmpstvwz] ::lc2 zho
+::s1 m ::s2 ::cost 1
+::s1 n ::s2 ::cost 1
+::s1 n ::s2 ::cost 0.7 ::right1 [-,$ ]
+::s1 o ::s2 ::cost 0.1
+::s1 p ::s2 ::cost 1
+::s1 q ::s2 ::cost 1
+::s1 r ::s2 ::cost 1
+::s1 r ::s2 ::cost 0.5 ::left1 /[aou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ]
+::s1 r ::s2 ::cost 0.3 ::left1 /[aeiou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ] ::lc2 zho
+::s1 re ::s2 ::cost 0.4 ::left1 /[ou]$/ ::right1 [-,$ ] ::lc2 zho
+::s1 re ::s2 ::cost 0.5 ::left1 /[aeiou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ] ::lc2 zho
+::s1 rr ::s2 ::cost 0.5 ::left1 /[aeiou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ] ::lc2 zho
+::s1 s ::s2 ::cost 1
+::s1 s ::s2 ::cost 0.6 ::right1 [-,$ ]
+::s1 t ::s2 ::cost 1
+::s1 t ::s2 ::cost 0.5 ::left1 /[aeiou][lnr]?$/ ::right1 [-,$ ]
+::s1 t ::s2 ::cost 0.6 ::left1 /[bcdfghklmnpqrstvwxz]$/ ::right1 [bcdfghklmnpqrstvwxz]
+::s1 u ::s2 ::cost 0.1
+::s1 v ::s2 ::cost 1
+::s1 w ::s2 ::cost 1
+::s1 w ::s2 ::cost 0.4 ::lc1 eng ::right1 [i][c][hk][-,$ ] ::example Greenwich, Alnwick
+::s1 x ::s2 ::cost 1
+::s1 y ::s2 ::cost 0.3
+::s1 z ::s2 ::cost 1
+::s1 ı ::s2 ::cost 0.3
+::s1 0 ::s2 ::cost 1
+::s1 1 ::s2 ::cost 1
+::s1 2 ::s2 ::cost 1
+::s1 3 ::s2 ::cost 1
+::s1 4 ::s2 ::cost 1
+::s1 5 ::s2 ::cost 1
+::s1 6 ::s2 ::cost 1
+::s1 7 ::s2 ::cost 1
+::s1 8 ::s2 ::cost 1
+::s1 9 ::s2 ::cost 1
+::s1 ' ::s2 ::cost 0.1
+::s1 ` ::s2 ::cost 0.1
+::s1 ( ::s2 ::cost 0.1
+::s1 ) ::s2 ::cost 0.1
+::s1 , ::s2 ::cost 0.1
+::s1 ; ::s2 ::cost 0.1
+::s1 - ::s2 ::cost 0.1
+::s1 . ::s2 ::cost 0.1
+::s1 .. ::s2 ::cost 0.12
+::s1 ... ::s2 ::cost 0.14
+::s1 ? ::s2 ::cost 0.2
+::s1 ! ::s2 ::cost 0.2
+::s1 ‼ ::s2 ::cost 0.2
+::s1 ‼ ::s2 !! ::cost 0.02
+::s1 ‼ ::s2 ! ::cost 0.1
+::s1 / ::s2 ::cost 0.1
+::s1 : ::s2 ::cost 0.1
+::s1 ː ::s2 ::cost 0.1
+::s1 ː ::s2 : ::cost 0.1
+::s1 « ::s2 ::cost 0.1
+::s1 » ::s2 ::cost 0.1
+::s1 – ::s2 ::cost 0.1
+::s1 – ::s2 - ::cost 0.05
+::s1 — ::s2 ::cost 0.15
+::s1 — ::s2 - ::cost 0.1
+::s1 — ::s2 – ::cost 0.05
+::s1 ─ ::s2 ::cost 0.2
+::s1 ─ ::s2 - ::cost 0.15
+::s1 ─ ::s2 – ::cost 0.1
+::s1 ─ ::s2 — ::cost 0.05
+::s1 ’ ::s2 ::cost 0.1
+::s1 ʼ ::s2 ::cost 0.1
+::s1 " " ::s2 ::cost 0.1
+::s1 “ ::s2 ::cost 0.1
+::s1 ” ::s2 ::cost 0.1
+::s1 ″ ::s2 ::cost 0.1
+::s1 # ::s2 ::cost 0.3
+::s1 + ::s2 ::cost 0.3
+::s1 * ::s2 ::cost 0.3
+::s1 = ::s2 ::cost 0.3
+::s1 < ::s2 ::cost 0.3
+::s1 > ::s2 ::cost 0.3
+::s1 [ ::s2 ::cost 0.3
+::s1 ] ::s2 ::cost 0.3
+::s1 { ::s2 ::cost 0.3
+::s1 } ::s2 ::cost 0.3
+::s1 | ::s2 ::cost 0.3
+::s1 & ::s2 ::cost 0.3
+::s1 _ ::s2 ::cost 0.3
+::s1 • ::s2 ::cost 0.1
+::s1 · ::s2 ::cost 0.1
+::s1 ◦ ::s2 ::cost 0.1
+::s1 ° ::s2 ::cost 0.1
+::s1 … ::s2 ::cost 0.1
+::s1 … ::s2 ... ::cost 0
+::s1 @ ::s2 ::cost 0.3
+::s1 © ::s2 ::cost 0.3
+::s1 © ::s2 (c) ::cost 0.1
+
+
+::s1 a ::s2 aa ::cost 0.02
+::s1 a ::s2 aaa ::cost 0.03
+::s1 a ::s2 aaaa ::cost 0.03
+::s1 a ::s2 aaaaa ::cost 0.03
+::s1 a ::s2 aaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaaaaa ::cost 0.04
+::s1 b ::s2 bb ::cost 0.02
+::s1 b ::s2 bbb ::cost 0.03
+::s1 b ::s2 bbbb ::cost 0.03
+::s1 b ::s2 bbbbb ::cost 0.03
+::s1 c ::s2 cc ::cost 0.02
+::s1 c ::s2 ccc ::cost 0.03
+::s1 c ::s2 cccc ::cost 0.03
+::s1 c ::s2 ccccc ::cost 0.03
+::s1 d ::s2 dd ::cost 0.02
+::s1 d ::s2 ddd ::cost 0.03
+::s1 d ::s2 dddd ::cost 0.03
+::s1 d ::s2 ddddd ::cost 0.03
+::s1 e ::s2 ee ::cost 0.02
+::s1 e ::s2 eee ::cost 0.03
+::s1 e ::s2 eeee ::cost 0.03
+::s1 e ::s2 eeeee ::cost 0.03
+::s1 e ::s2 eeeeee ::cost 0.04
+::s1 e ::s2 eeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeeeeee ::cost 0.04
+::s1 f ::s2 ff ::cost 0.02
+::s1 f ::s2 fff ::cost 0.03
+::s1 f ::s2 ffff ::cost 0.03
+::s1 f ::s2 fffff ::cost 0.03
+::s1 g ::s2 gg ::cost 0.02
+::s1 g ::s2 ggg ::cost 0.03
+::s1 g ::s2 gggg ::cost 0.03
+::s1 g ::s2 ggggg ::cost 0.03
+::s1 h ::s2 hh ::cost 0.02
+::s1 h ::s2 hhh ::cost 0.03
+::s1 h ::s2 hhhh ::cost 0.03
+::s1 h ::s2 hhhhh ::cost 0.03
+::s1 i ::s2 ii ::cost 0.02
+::s1 i ::s2 iii ::cost 0.03
+::s1 i ::s2 iiii ::cost 0.03
+::s1 i ::s2 iiiii ::cost 0.03
+::s1 i ::s2 iiiiii ::cost 0.04
+::s1 i ::s2 iiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiiiiii ::cost 0.04
+::s1 j ::s2 jj ::cost 0.02
+::s1 j ::s2 jjj ::cost 0.03
+::s1 j ::s2 jjjj ::cost 0.03
+::s1 j ::s2 jjjjj ::cost 0.03
+::s1 k ::s2 kk ::cost 0.02
+::s1 k ::s2 kkk ::cost 0.03
+::s1 k ::s2 kkkk ::cost 0.03
+::s1 k ::s2 kkkkk ::cost 0.03
+::s1 l ::s2 ll ::cost 0.02
+::s1 l ::s2 lll ::cost 0.03
+::s1 l ::s2 llll ::cost 0.03
+::s1 l ::s2 lllll ::cost 0.03
+::s1 m ::s2 mm ::cost 0.02
+::s1 m ::s2 mmm ::cost 0.03
+::s1 m ::s2 mmmm ::cost 0.03
+::s1 m ::s2 mmmmm ::cost 0.03
+::s1 n ::s2 nn ::cost 0.02
+::s1 n ::s2 nnn ::cost 0.03
+::s1 n ::s2 nnnn ::cost 0.03
+::s1 n ::s2 nnnnn ::cost 0.03
+::s1 o ::s2 oo ::cost 0.02
+::s1 o ::s2 ooo ::cost 0.03
+::s1 o ::s2 oooo ::cost 0.03
+::s1 o ::s2 ooooo ::cost 0.03
+::s1 o ::s2 oooooo ::cost 0.04
+::s1 o ::s2 ooooooo ::cost 0.04
+::s1 o ::s2 oooooooo ::cost 0.04
+::s1 o ::s2 ooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooo ::cost 0.04
+::s1 o ::s2 ooooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooooo ::cost 0.04
+::s1 o ::s2 ooooooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooooooo ::cost 0.04
+::s1 o ::s2 ooooooooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooooooooo ::cost 0.04
+::s1 p ::s2 pp ::cost 0.02
+::s1 p ::s2 ppp ::cost 0.03
+::s1 p ::s2 pppp ::cost 0.03
+::s1 p ::s2 ppppp ::cost 0.03
+::s1 q ::s2 qq ::cost 0.02
+::s1 q ::s2 qqq ::cost 0.03
+::s1 q ::s2 qqqq ::cost 0.03
+::s1 q ::s2 qqqqq ::cost 0.03
+::s1 r ::s2 rr ::cost 0.02
+::s1 r ::s2 rrr ::cost 0.03
+::s1 r ::s2 rrrr ::cost 0.03
+::s1 r ::s2 rrrrr ::cost 0.03
+::s1 s ::s2 ss ::cost 0.02
+::s1 s ::s2 sss ::cost 0.03
+::s1 s ::s2 ssss ::cost 0.03
+::s1 s ::s2 sssss ::cost 0.03
+::s1 t ::s2 tt ::cost 0.02
+::s1 t ::s2 ttt ::cost 0.03
+::s1 t ::s2 tttt ::cost 0.03
+::s1 t ::s2 ttttt ::cost 0.03
+::s1 u ::s2 uu ::cost 0.02
+::s1 u ::s2 uuu ::cost 0.03
+::s1 u ::s2 uuuu ::cost 0.03
+::s1 u ::s2 uuuuu ::cost 0.03
+::s1 u ::s2 uuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuuuuu ::cost 0.04
+::s1 v ::s2 vv ::cost 0.02
+::s1 v ::s2 vvv ::cost 0.03
+::s1 v ::s2 vvvv ::cost 0.03
+::s1 v ::s2 vvvvv ::cost 0.03
+::s1 w ::s2 ww ::cost 0.02
+::s1 w ::s2 www ::cost 0.03
+::s1 w ::s2 wwww ::cost 0.03
+::s1 w ::s2 wwwww ::cost 0.03
+::s1 x ::s2 xx ::cost 0.02
+::s1 x ::s2 xxx ::cost 0.03
+::s1 x ::s2 xxxx ::cost 0.03
+::s1 x ::s2 xxxxx ::cost 0.03
+::s1 y ::s2 yy ::cost 0.02
+::s1 y ::s2 yyy ::cost 0.03
+::s1 y ::s2 yyyy ::cost 0.03
+::s1 y ::s2 yyyyy ::cost 0.03
+::s1 z ::s2 zz ::cost 0.02
+::s1 z ::s2 zzz ::cost 0.03
+::s1 z ::s2 zzzz ::cost 0.03
+::s1 z ::s2 zzzzz ::cost 0.03
+::s1 " " ::s2 "  " ::cost 0
+::s1 . ::s2 ::left1 /\./ ::left2 /\./  ::cost 0.02
+::s1 … ::s2 ::left1 /…/ ::left2 /…/  ::cost 0.01
+::s1 _ ::s2 ::left1 /_/ ::left2 /_/ ::cost 0.01
+::s1 = ::s2 ::left1 /=/ ::left2 /=/ ::cost 0.01
+::s1 ! ::s2 ::left1 /!/ ::left2 /!/ ::cost 0.02
+::s1 ? ::s2 ::left1 /\?/ ::left2 /\?/ ::cost 0.02
+::s1 aa ::s2 aː ::cost 0.02
+::s1 ee ::s2 eː ::cost 0.02
+::s1 ii ::s2 iː ::cost 0.02
+::s1 oo ::s2 oː ::cost 0.02
+::s1 uu ::s2 uː ::cost 0.02
+
+::s1 a ::s2 e ::cost 0.1
+::s1 au ::s2 o ::cost 0.1 ::lc1 eng
+::s1 aw ::s2 o ::cost 0.3 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 aw ::s2 o ::cost 0.1 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 aw ::s2 a ::cost 0.2 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 ay ::s2 i ::cost 0.02 ::lc1 fas ::lc2 eng
+::s1 aye ::s2 ae ::cost 0.05 ::lc1 fas
+::s1 é ::s2 e ::cost 0.05
+::s1 e ::s2 i ::cost 0.15
+::s1 e ::s2 i ::cost 0.1 ::lc1 uig ::lc2 uig
+::s1 e ::s2 y ::cost 0.15
+::s1 ew ::s2 u ::cost 0.3 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ew ::s2 u ::cost 0.1 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 ew ::s2 u ::cost 0.3 ::right1 [aei][lgnrst] ::lc1 eng
+::s1 ew ::s2 e ::cost 0.3 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 i ::s2 a ::cost 0.1 ::right1 [-,$ ] ::lc1 fas
+::s1 i ::s2 ea ::cost 0.03 ::lc2 eng
+::s1 i ::s2 ee ::cost 0.03 ::lc2 eng
+::s1 i ::s2 ei ::cost 0.05 ::lc2 eng
+::s1 i ::s2 ie ::cost 0.03 ::lc2 eng
+::s1 i ::s2 ı ::cost 0.05 
+::s1 i ::s2 e ::cost 0.1 ::lc2 eng
+::s1 i ::s2 y ::cost 0.15
+::s1 i ::s2 y ::cost 0.1 ::right2 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ie ::s2 ei ::cost 0.15
+::s1 ie ::s2 y ::cost 0.15
+::s1 ij ::s2 ai ::cost 0.15
+::s1 o ::s2 u ::cost 0.1
+::s1 oo ::s2 u ::cost 0.1
+::s1 ow ::s2 au ::cost 0.2 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ow ::s2 o ::cost 0.2 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ow ::s2 o ::cost 0.2 ::lc1 eng ::lc2 zho ::right1 [e]
+::s1 ow ::s2 o ::cost 0.4 ::lc1 eng ::lc2 zho ::right1 [iy]
+::s1 u ::s2 a ::cost 0.1 ::lc1 eng ::right1 [-,bcdfghklmnpqrstvwxz][bcdfghklmnpqrstvwxz$ ]
+::s1 u ::s2 ou ::cost 0.05
+::s1 u ::s2 yu ::cost 0.05 ::left1 /^(.*[- ])?$/
+::s1 yeo ::s2 eo ::cost 0.1 ::lc1 fas
+
+# Amharic
+::s1 a ::s2 e ::cost 0.05 ::lc1 amh
+::s1 aa ::s2 o ::cost 0.15 ::lc1 amh
+::s1 aawe ::s2 au ::cost 0.05 ::lc1 amh
+::s1 aawe ::s2 ao ::cost 0.1 ::lc1 amh
+::s1 aawe ::s2 ou ::cost 0.1 ::lc1 amh
+::s1 aawo ::s2 ao ::cost 0.05 ::lc1 amh
+::s1 aaye ::s2 ai ::cost 0.05 ::lc1 amh
+::s1 aaye ::s2 i ::cost 0.1 ::lc1 amh
+::s1 aaye ::s2 ei ::cost 0.1 ::lc1 amh
+::s1 awe ::s2 au ::cost 0.05 ::lc1 amh
+::s1 awe ::s2 ao ::cost 0.1 ::lc1 amh
+::s1 awe ::s2 ou ::cost 0.1 ::lc1 amh
+::s1 ee ::s2 ai ::cost 0.1 ::lc1 amh
+::s1 eewo ::s2 eo ::cost 0.05 ::lc1 amh
+::s1 eeyaa ::s2 ea ::cost 0.1 ::lc1 amh
+::s1 eeye ::s2 ai ::cost 0.1 ::lc1 amh
+::s1 ewee ::s2 ue ::cost 0.1 ::lc1 amh
+::s1 gwaa ::s2 gua ::cost 0.05 ::lc1 amh
+::s1 iya ::s2 ie  ::cost 0.05 ::lc1 amh
+::s1 iyaa ::s2 ia ::cost 0.05 ::lc1 amh
+::s1 iyo ::s2 io ::cost 0.05 ::lc1 amh
+::s1 kxaa ::s2 kha ::cost 0.05 ::lc1 amh
+::s1 liyaa ::s2 llia ::cost 0.05 ::lc1 amh
+::s2 qaa ::s2 cca ::cost 0.05 ::lc1 amh
+::s1 uwaa ::s2 ua ::cost 0.05 ::lc1 amh
+::s1 uwee ::s2 ue ::cost 0.05 ::lc1 amh
+::s1 uwi ::s2 oui ::cost 0.05 ::lc1 amh
+::s1 uwi ::s2 ui ::cost 0.05 ::lc1 amh
+::s1 xaaye ::s2 hai ::cost 0.1 ::lc1 amh
+::s1 xwaa ::s2 jua ::cost 0.1 ::lc1 amh
+::s1 ziyaa ::s1 sia ::cost 0.05 ::lc1 amh
+::s1 w ::s2 ::cost 0.3 ::lc1 amh ::left1 /[aeiou]$/ ::right1 [aeiou]
+::s1 y ::s2 ::cost 0.1 ::lc1 amh ::left1 /[aeiou]$/ ::right1 [aeiou]
+# abbreviations
+::s1 ee. ::s2 a ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 si. ::s2 c ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 di. ::s2 d ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 eefe. ::s2 f ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 are. ::s2 r ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+
+# Arabic
+::s1 ::s2 a ::cost 0.02 ::lc1 ara
+::s1 ::s2 e ::cost 0.02 ::lc1 ara
+::s1 ::s2 i ::cost 0.05 ::lc1 ara
+::s1 ::s2 o ::cost 0.05 ::lc1 ara
+::s1 ::s2 p ::cost 0.15 ::lc1 ara ::left2 /m$/ ::right2 [dfgklmnpqrstvwz]
+::s1 ::s2 u ::cost 0.05 ::lc1 ara
+::s1 y ::s2 a ::cost 0.15 ::lc1 ara
+::s1 y ::s2 e ::cost 0.05 ::lc1 ara
+::s1 y ::s2 ea ::cost 0.02 ::lc1 ara
+::s1 y ::s2 ee ::cost 0.02 ::lc1 ara
+::s1 y ::s2 i ::cost 0.02 ::lc1 ara
+::s1 y ::s2 ie ::cost 0.02 ::lc1 ara
+::s1 b ::s2 p ::cost 0.02 ::lc1 ara
+::s1 b ::s2 pp ::cost 0.03 ::lc1 ara
+::s1 f ::s2 v ::cost 0.02 ::lc1 ara
+::s1 fyl ::s2 ville ::right2 [-,$ ] ::cost 0.05 ::lc1 ara
+::s1 gh ::s2 g ::right2 [abcdfgklmnopqrstuvwz] ::cost 0.05 ::lc1 ara
+::s1 ghz ::s2 gs ::cost 0.05 ::lc1 ara
+::s1 j ::s2 g ::cost 0.2 ::lc1 ara
+::s1 kh ::s2 g ::cost 0.3 ::lc1 ara ::right2 [eiy] 
+::s1 q ::s2 g ::cost 0.2 ::lc1 ara ::right2 [arouz]
+::s1 q ::s2 gg ::cost 0.2 ::lc1 ara ::right2 [arouz]
+::s1 th ::s2 z ::cost 0.4 ::lc1 ara ::right2 [aou] ::comment Spanish
+::s1 " (" ::s2 ", " ::cost 0.02 ::lc1 ara
+::s1 ) ::s2 ::right2 [-,$ ] ::cost 0.02 ::lc1 ara
+
+# Bengali
+::s1 aoyaa ::s2 wa ::cost 0.1 ::lc1 ben
+::s1 aoye ::s2 way ::cost 0.1 ::lc1 ben
+::s1 bhaa ::s2 ve ::cost 0.1 ::lc1 ben
+::s1 bh ::s2 v ::cost 0.2 ::lc1 ben
+::s1 bh ::s2 w ::cost 0.2 ::lc1 ben
+::s1 b ::s2 v ::cost 0.3 ::lc1 ben
+::s1 b ::s2 w ::cost 0.3 ::lc1 ben
+::s1 dda ::s2 rh ::right2 [-,$ ] ::cost 0.2 ::lc1 ben
+::s1 dd ::s2 r ::cost 0.4 ::lc1 ben
+::s1 gk ::s2 k ::cost 0.05 ::lc1 ben
+::s1 h ::s2 g ::right2 [eiy] ::cost 0.4 ::lc1 ben
+::s1 h ::s2 j ::cost 0.4 ::lc1 ben
+::s1 hoyaai ::s2 whi ::cost 0.05 ::lc1 ben
+::s1 j ::s2 z ::cost 0.1 ::lc1 ben
+::s1 j ::s2 s ::cost 0.3 ::lc1 ben
+::s1 myaaka ::s2 mc ::cost 0.1 ::lc1 ben
+::s1 myaaka ::s2 mac ::cost 0.1 ::lc1 ben
+::s1 oyaa ::s2 wa ::cost 0.02 ::lc1 ben
+::s1 oyaa ::s2 wo ::cost 0.1 ::lc1 ben
+::s1 oyena ::s2 owen ::cost 0.1 ::lc1 ben
+::s1 ph ::s2 v ::cost 0.1 ::lc1 ben
+::s1 phana ::s2 von ::cost 0.1 ::lc1 ben
+::s1 rhio ::s2 gio ::cost 0.2 ::lc1 ben
+::s1 sh ::s2 s ::cost 0.4 ::lc1 ben
+::s1 ss ::s2 sh ::left1 /[k]$/ ::cost 0.15 ::lc1 ben
+::s1 ss ::s2 sh ::cost 0.3 ::lc1 ben
+::s1 o ::s2 wo ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 oye ::s2 we ::cost 0.2 ::lc1 ben
+::s1 tta ::s2 tho ::cost 0.3 ::lc1 ben
+::s1 tthaa ::s2 ta ::cost 0.3 ::lc1 ben
+::s1 u ::s2 wo ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 u ::s2 woo ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 u ::s2 wu ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 ui ::s2 wi ::cost 0.02 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 yaa ::s2 wa ::cost 0.3 ::lc1 ben
+::s1 ye ::s2 we ::cost 0.3 ::lc1 ben
+
+# Russian
+::s1 ::s2 os ::cost 0.4 ::left2 /[bcdfghilmnprstvx]$/ ::right2 [-,$ ] ::lc1 rus
+::s1 ::s2 us ::cost 0.4 ::left2 /[bcdfghilmnprstvx]$/ ::right2 [-,$ ] ::lc1 rus
+::s1 av ::s2 au ::cost 0.05 ::lc1 rus
+::s1 ch ::s2 cz ::cost 0.1 ::lc1 rus ::comment Polish
+::s1 chch ::s2 cci ::right2 [aou] ::cost 0.1 ::lc1 rus
+::s1 chch ::s2 cc ::right2 [eiy] ::cost 0.1 ::lc1 rus
+::s1 chzh ::s2 zh ::cost 0.1 ::lc1 rus
+::s1 dz ::s2 zz ::cost 0.1 ::lc1 rus ::right2 [aeiouy]
+::s1 dz ::s2 j ::cost 0.3 ::lc1 rus ::right2 [aeiouy] ::comment Japanese
+::s1 dzh ::s2 g ::cost 0.05 ::lc1 rus ::right2 [eiy]
+::s1 dzh ::s2 gg ::cost 0.05 ::lc1 rus ::right2 [eiy]
+::s1 dzh ::s2 j ::cost 0.05 ::lc1 rus
+::s1 ev ::s2 eu ::cost 0.1 ::lc1 rus
+::s1 f ::s2 th ::cost 0.6 ::lc1 rus
+::s1 ievye ::s2 iaceae ::cost 0.02 ::right1 [-,$ ] ::lc1 rus ::comment scientific names for families of species 
+::s1 ii ::s2 ius ::cost 0.2 ::right1 [-,$ ] ::lc1 rus
+::s1 i ::s2 j ::cost 0.2 ::lc1 rus
+::s1 naya ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::suffix adjective
+::s1 nyi ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::suffix adjective
+::s1 ovye ::s2 aceae ::cost 0.02 ::right1 [-,$ ] ::lc1 rus ::comment scientific names for families of species
+::s1 shsh ::s2 sh ::cost 0 ::lc1 rus
+::s1 skaya ::s2 ian ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix possessive
+::s1 skaya ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix possessive
+::s1 skii ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix possessive
+::s1 skii ::s2 ian ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix adjective
+::s1 tsian ::s2 tian ::cost 0.05 ::lc1 rus
+::s1 tsion ::s2 tion ::cost 0.05 ::lc1 rus
+::s1 ts ::s2 c ::cost 0.3 ::lc1 rus
+::s1 ts ::s2 c ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 tsz ::s2 z ::cost 0.1 ::lc1 rus
+::s1 itsa ::s2 ica ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 etski ::s2 ecky ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 tsiya ::s2 tion ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 tsi ::s2 qi ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 tsy ::s2 qi ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 tszi ::s2 ji ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 tszy ::s2 ji ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 u ::s2 w ::right2 [aeio] ::cost 0.05 ::lc1 rus
+::s1 u ::s2 w ::cost 0.2 ::lc1 rus
+::s1 uo ::s2 wa ::cost 0.2 ::lc1 rus ::right2 [lnrst]
+::s1 v ::s2 u ::cost 0.05 ::lc1 rus ::left1 /[bcdfghjklmnpqrstvwxz]$/ ::right1 [aeiou] 
+::s1 gva ::s2 gua ::cost 0.02 ::lc1 rus
+::s1 gvi ::s2 gui ::cost 0.02 ::lc1 rus
+::s1 x ::s2 sh ::cost 0.2 ::left2 /[aeiou]$/ ::right2 [-,aouct$-] ::lc1 rus
+::s1 y ::s2 s ::cost 0.4 ::right2 [-,$-] ::lc1 rus
+::s1 zh ::s2 rz ::cost 0.1 ::lc1 rus ::comment Polish rz
+
+# Russian case endings
+::s1 em ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ey ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 om ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 oy ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 oyu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 y ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ya ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ye ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 yem ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ym ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ymi ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 yu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ii ::s2 iya ::cost 0.1 ::right1 [-,$ ] ::right2 [-,$ ] ::lc1 rus ::lc2 rus ::comment Russian case endings
+::s1 ii ::s2 iye ::cost 0.1 ::right1 [-,$ ] ::right2 [-,$ ] ::lc1 rus ::lc2 rus ::comment Russian case endings
+
+::s1 am ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ami ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 em ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ev ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 eri ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 eryu ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 om ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ov ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 akh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ykh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+
+# Ukrainian case endings
+::s1 eyu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 oyu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 ya ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yi ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+
+::s1 am ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 amy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 em ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 evy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 iv ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 om ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 ovy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yam ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yamy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yiv ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 akh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yakh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+
+# Uyghur
+::s1 aw ::s2 ao ::cost 0.05 ::lc1 uig
+::s1 aw ::s2 au ::cost 0.05 ::lc1 uig
+::s1 gwi ::s2 gui ::cost 0.05 ::lc1 uig
+::s1 iye ::s2 ia ::cost 0.05 ::lc1 uig
+::s1 istan ::s2 ia ::cost 0.1 ::right1 [-,$ ] ::lc1 uig
+::s1 j ::s2 c ::cost 0.4 ::lc1 uig
+::s1 q ::s2 h ::cost 0.2 ::lc1 uig
+::s1 sey ::s2 cai ::cost 0.2 ::lc1 uig
+::s1 sh ::s2 x ::cost 0.2 ::lc1 uig
+
+::s1 b ::s2 p ::cost 0.3
+::s1 b ::s2 v ::cost 0.5 ::left2 /^(.*[- ])?$/
+::s1 b ::s2 v ::cost 0.7
+::s1 c ::s2 ch ::cost 0.25 ::right1 [eiy]
+::s1 c ::s2 ck ::cost 0.02 ::right1 [-,abcdfghklmnpoqrstuvwxz$ ]
+::s1 c ::s2 k ::cost 0.4
+::s1 c ::s2 k ::cost 0.05 ::left1 /^(.* )?ma?$/ ::comment MacIntyre
+::s1 c ::s2 k ::cost 0.02 ::right1 [-,abcdfghklmnpoqrstuvwxz$ ]
+::s1 c ::s2 kk ::cost 0.02 ::right1 [-,abcdfghklmnpoqrstuvwxz$ ]
+::s1 c ::s2 s ::cost 0.7
+::s1 c ::s2 s ::cost 0.1 ::right1 [eiy]
+::s1 c ::s2 ts ::cost 0.15 ::right1 [eiy]
+::s1 c ::s2 z ::cost 0.3
+::s1 ch ::s2 ck ::cost 0.2
+::s1 ch ::s2 g ::cost 0.3 ::right1 [eiy] ::right2 [eiy]
+::s1 ch ::s2 k ::cost 0.2
+::s1 ch ::s2 kk ::cost 0.2
+::s1 ch ::s2 sh ::cost 0.3
+::s1 ch ::s2 sh ::cost 0.2 ::left1 /eiy$/ ::right1 [$ ]
+::s1 ch ::s2 tch ::cost 0.1
+::s1 ch ::s2 tsh ::cost 0.1
+::s1 ch ::s2 z ::cost 0.5
+::s1 ck ::s2 kk ::cost 0.02
+::s1 cz ::s2 ch ::cost 0.2 ::left1 /i$/
+::s1 d ::s2 t ::cost 0.3
+::s1 de ::s2 dre ::cost 0.3 ::lc1 zho ::right2 [-,$ ]
+::s1 dg ::s2 j ::cost 0.6 ::lc1 eng ::comment Cambridge
+::s1 dg ::s2 j ::cost 0.3 ::right1 [eiy] ::lc1 eng
+::s1 dg ::s2 j ::cost 0.1 ::right1 [eiy] ::lc1 eng ::lc2 fas, jpn
+::s1 dt ::s2 d ::cost 0.3
+::s1 dt ::s2 t ::cost 0.03
+::s1 dt ::s2 tt ::cost 0.03
+::s1 f ::s2 p ::cost 0.8
+::s1 f ::s2 ph ::cost 0.01
+::s1 ff ::s2 ph ::cost 0.02
+::s1 f ::s2 pf ::cost 0.1
+::s1 f ::s2 v ::cost 0.3
+::s1 f ::s2 v ::cost 0.1  ::right1 [-,$ ]
+::s1 ef ::s2 ev ::cost 0.1 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 f ::s2 w ::cost 0.3
+::s1 g ::s2 j ::cost 0.6
+::s1 g ::s2 j ::cost 0.3 ::right1 [eiy]
+::s1 g ::s2 j ::cost 0.1 ::right1 [eiy] ::lc2 amh, ara, fas, jpn, som
+::s1 g ::s2 k ::cost 0.3
+::s1 g ::s2 gh ::cost 0.3
+::s1 g ::s2 ch ::cost 0.4 ::left1 /[eiy]$/ ::right1 [-,$ ] ::comment German: Ludwig, Braunschweig
+::s1 gh ::s2 f ::cost 0.2 ::lc1 eng ::comment laughter
+::s1 gh ::s2 "" ::cost 0.2 ::lc1 eng ::comment daughter
+::s1 gh ::s2 g ::cost 0.2 ::lc1 eng ::comment Afghanistan
+::s1 gl ::s2 l ::cost 0.2 ::lc1 eng ::right1 [i]
+::s1 gn ::s2 n ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 gn ::s2 n ::cost 0.2 ::lc1 eng
+::s1 gz ::s2 ks ::cost 0.2
+::s1 h ::s2 e ::cost 0.4 ::lc1 fas
+::s1 ise ::s2 ize ::cost 0.1
+::s1 j ::s2 y ::cost 0.2
+::s1 j ::s2 dj ::cost 0.2
+::s1 j ::s2 h ::cost 0.4 ::right2 [aeiou] ::lc2 amh ::example Jose
+::s1 j ::s2 hh ::cost 0.4 ::right2 [aeiou] ::lc2 amh ::example Tardajos
+::s1 j ::s2 zh ::cost 0.2
+::s1 k ::s2 cc ::cost 0.02 ::right2 [aour]
+::s1 k ::s2 cc ::cost 0.3
+::s1 k ::s2 cch ::cost 0.15
+::s1 k ::s2 ck ::cost 0.02
+::s1 k ::s2 cq ::cost 0.05
+::s1 k ::s2 cqu ::cost 0.05
+::s1 k ::s2 cque ::cost 0.1
+::s1 k ::s2 cque ::cost 0.05 ::right2 [-,$ ]
+::s1 k ::s2 cques ::cost 0.05 ::right2 [-,$ ]
+::s1 k ::s2 q ::cost 0.05
+::s1 k ::s2 qu ::cost 0.05
+::s1 k ::s2 que ::cost 0.1
+::s1 k ::s2 que ::cost 0.05 ::right2 [-,$ ]
+::s1 k ::s2 ques ::cost 0.1 ::right2 [-,$ ]
+::s1 kh ::s2 j ::cost 0.2
+::s1 kh ::s2 q ::cost 0.2
+::s1 kh ::s2 k ::cost 0.25 ::right1 [aeiouy]
+::s1 kh ::s2 k ::cost 0.1 ::right1 [aeiouys] ::lc2 amh
+::s1 kn ::s2 n ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 kj ::s2 sh ::cost 0.2 ::comment Swedish
+::s1 l ::s2 r ::cost 0.1 ::lc1 zho
+::s1 aib ::s2 alb ::cost 0.1 ::lc1 zho
+::s1 al ::s2 ::cost 0.5 ::left1 /^(.* )?$/
+::s1 al- ::s2 ::cost 0.3 ::left1 /^(.* )?$/
+::s1 el ::s2 ::cost 0.5 ::left1 /^(.* )?$/
+::s1 el- ::s2 ::cost 0.3 ::left1 /^(.* )?$/
+::s1 ll ::s2 y ::cost 0.1 ::left1 /[aeiouy]$/ ::right1 [aeiouy] ::comment Guillermo, Guillaume 
+::s1 mb ::s2 m ::cost 0.2 ::right1 [-,bcdfghklmnpqstvwxz$ ] ::lc1 eng ::comment bomb
+::s1 n ::s2 m ::cost 0.5 ::left1 /[aeiou]$/ ::left2 /[aeiou]$/ ::right1 [bcdfghklmnpqrstvwxz$ ] ::right2 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ng ::s2 n ::cost 0.1 ::left1 /[aeiou]$/ ::lc1 zho
+::s1 ng ::s2 m ::cost 0.25 ::left1 /[aeiou]$/ ::lc1 zho
+::s1 ng ::s2 n ::cost 0.1 ::left2 /[aeiou]$/ ::lc2 ara, ben, rus, zho
+::s1 nm ::s2 m ::cost 0.25 ::lc1 zho ::left1 
+::s1 pn ::s2 n ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 ph ::s2 p ::cost 0.3 ::lc1 amh
+::s1 q ::s2 c ::cost 0.15
+::s1 q ::s2 ch ::cost 0.2 ::right2 [eiy]
+::s1 q ::s2 ck ::cost 0.2
+::s1 q ::s2 kk ::cost 0.2
+::s1 q ::s2 gh ::cost 0.2 ::lc1 fas ::right2 [aeiouy]
+::s1 qi ::s2 ch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 cci ::cost 0.1 ::lc1 zho
+::s1 qi ::s2 chi ::cost 0.1 ::lc1 zho
+::s1 qi ::s2 tch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 ts ::cost 0.4 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 tsch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 tzsch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 czy ::cost 0.2 ::lc1 zho
+::s1 qu ::s2 kw ::cost 0.15
+::s1 qu ::s2 kv ::cost 0.15
+::s1 e ::s2 er ::cost 0.25 ::left1 /[bcdfghklmnpqrstvwxz]$/ ::lc1 zho
+::s1 re ::s2 er ::cost 0.1
+::s1 rh ::s2 r ::cost 0.05 ::left1 /^(.*[- ])?$/ ::example Rhine
+::s1 s ::s2 sh ::cost 0.03 ::right2 [aeiou] ::lc2 amh
+::s1 s ::s2 sz ::cost 0.3 ::lc2 eng ::example Liszt (Hungarian)
+::s1 s ::s2 ts ::cost 0.4 ::lc1 amh, zho
+::s1 s ::s2 z ::cost 0.4
+::s1 s ::s2 z ::cost 0.1 ::left1 /[aeiouy]$/ ::right1 [aeiouy] ::lc1 eng
+::s1 s ::s2 z ::cost 0.1 ::left1 /[aeiouy][bdglmnrvw]?$/ ::right1 [-,$ ] ::lc1 eng
+::s1 s ::s2 z ::cost 0.2 ::lc2 fas
+::s1 sc ::s2 s ::cost 0.2 ::right1 [i] ::example Nascimento
+::s1 sci ::s2 sh ::cost 0.2 ::example Brescia
+::s1 sch ::s2 sh ::cost 0.1
+::s1 sh ::s2 sz ::cost 0.2 ::example Mariusz (Polish) ::lc2 eng
+::s1 si ::s2 j ::cost 0.1 ::right2 [a] ::lc1 eng
+::s1 ss ::s2 z ::cost 0.5
+# ::s1 smith ::s2 mith ::cost 0.75 ::lc2 zho ::comment weird, but several different Xinhua examples
+::s1 tch ::s2 c ::cost 0.2 ::left2 /[aeiou]$/ ::right2 [-,e$ ]
+::s1 te ::s2 tre ::cost 0.3 ::lc1 zho ::right2 [-,$ ]
+::s1 th ::s2 t ::cost 0.2 ::lc2 amh, fas, uig
+::s1 th ::s2 s ::cost 0.4 ::lc2 zho
+::s1 th ::s2 sth ::cost 0.4 ::lc1 zho
+::s1 th ::s2 ths ::cost 0.4 ::lc1 zho
+::s1 th ::s2 z ::cost 0.3 ::lc2 amh ::right2 [-,$ aeot]
+::s1 v ::s2 w ::cost 0.02
+::s1 v ::s2 wh ::cost 0.02 ::left1 /^(.* )?$/
+::s1 vv ::s2 w ::cost 0.02
+::s1 w ::s2 u ::cost 0.1 ::lc2 uig
+::s1 wa ::s2 ua ::cost 0.05
+::s1 wh ::s2 w ::cost 0.05 ::left1 /^(.* )?$/
+::s1 wr ::s2 r ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 x ::s2 ks ::cost 0.05
+::s1 x ::s2 s ::cost 0.2 ::left1 /^(.* )?$/
+::s1 x ::s2 sh ::cost 0.2 ::lc1 uig ::left1 /^(.* )?$/ ::right1 [aeiou]
+::s1 x ::s2 z ::cost 0.2 ::left1 /^(.* )?$/ ::right1 [aeiouy]
+::s1 x ::s2 h ::cost 0.3 ::lc1 uig
+::s1 x ::s2 h ::cost 0.05 ::lc1 uig ::left1 /^(.* )?$/ ::right1 [aeiou]
+::s1 x ::s2 kh ::cost 0.1 ::lc1 uig
+::s1 xi ::s2 sch ::cost 0.2 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 sh ::cost 0.2 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 ch ::cost 0.4 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 sci ::cost 0.4 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 s ::cost 0.6 ::right1 [aeou] ::lc1 zho
+::s1 z ::s2 dz ::cost 0.1 ::left1 /^(.*[ aeiouy])?[lnr]?$/
+::s1 z ::s2 ts ::cost 0.15
+::s1 z ::s2 tz ::cost 0.15
+::s1 zh ::s2 g ::cost 0.2 ::right2 [eiy]
+::s1 zh ::s2 g ::cost 0.1 ::right2 [eiy] ::lc2 amh
+::s1 zz ::s2 ts ::cost 0.15
+::s1 zz ::s2 tz ::cost 0.1
+
+# Oromo
+::s1 nb ::s2 mb ::cost 0.4 ::lc1 orm ::lc2 orm ::left1 /[aeiou]$/ ::left2 /[aeiou]$/
+::s1 np ::s2 mp ::cost 0.4 ::lc1 orm ::lc2 orm ::left1 /[aeiou]$/ ::left2 /[aeiou]$/
+::s1 ph ::s2 p ::cost 0.3 ::lc1 orm ::lc2 orm
+
+# Tigrinya
+::s1 aaye ::s2 a ::cost 0.4 ::lc1 tir ::lc2 tir ::left1 /[bcdfghklmnpqrstvwxz]$/ ::right1 [bcdfghklmnpqrstvwxz] ::comment internal plural
+::s1 aaye ::s2 i ::cost 0.4 ::lc1 tir ::lc2 tir ::left1 /[bcdfghklmnpqrstvwxz]$/ ::right1 [bcdfghklmnpqrstvwxz] ::comment internal plural
+
+# Somali
+::s1 ay ::s2 ey ::cost 0.1 ::lc1 som ::lc2 som
+::s1 ay ::s2 eey ::cost 0.15 ::lc1 som ::lc2 som
+::s1 aha ::s2 ihii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aha ::s2 ihi ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aha ::s2 uhu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ihii ::s2 uhu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ihi ::s2 uhu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ha ::s2 hii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ha ::s2 hi ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ha ::s2 hu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 hii ::s2 hu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 hi ::s2 hu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aka ::s2 ikii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aka ::s2 iki ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aka ::s2 uku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ikii ::s2 uku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 iki ::s2 uku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ka ::s2 kii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ka ::s2 ki ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ka ::s2 ku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 kii ::s2 ku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ki ::s2 ku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aga ::s2 ugu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ga ::s2 gu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 itii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 iti ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 utu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 itii ::s2 utu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 iti ::s2 utu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ta ::s2 tii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ta ::s2 ti ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ta ::s2 tu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 tii ::s2 tu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ti ::s2 tu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 ete ::cost 0.15 ::lc1 som ::lc2 som
+::s1 ata ::s2 iti ::cost 0.2 ::lc1 som ::lc2 som
+::s1 ete ::s2 iti ::cost 0.15 ::lc1 som ::lc2 som
+::s1 g ::s2 k ::cost 0.2 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [aeiou]
+::s1 g ::s2 k ::cost 0.25 ::lc1 som ::lc2 som
+::s1 g ::s2 kh ::cost 0.25 ::lc1 som ::lc2 som
+::s1 gh ::s2 kh ::cost 0.1 ::lc1 som ::lc2 som
+::s1 gh ::s2 k ::cost 0.2 ::lc1 som ::lc2 som
+::s1 g ::s2 q ::cost 0.25 ::lc1 som ::lc2 som
+::s1 g ::s2 q ::cost 0.2 ::lc1 som ::lc2 som ::right1 [aou] ::right2 [aou]
+::s1 ga ::s2 q ::cost 0.2 ::lc1 som ::lc2 som ::left1 /^(.*[aeiou])?$/ ::left2 /^(.*[aeiou])?$/ ::right1 [bcdfghklmnpqrstvwxz] ::right2 [bcdfghklmnpqrstvwxz]
+::s1 g ::s2 j ::cost 0.25 ::lc1 som ::lc2 som
+::s1 g ::s2 j ::cost 0.15 ::lc1 som ::lc2 som ::right1 [ei] ::right2 [ei]
+::s1 gi ::s2 j ::cost 0.15 ::lc1 som ::lc2 som ::right2 [ei]
+::s1 n ::s2 m ::cost 0.2 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [aeiou]
+::s1 n ::s2 mm ::cost 0.2 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [aeiou]
+::s1 n ::s2 m ::cost 0.25 ::lc1 som ::lc2 som ::right2 [aeiko]
+::s1 n ::s2 mm ::cost 0.25 ::lc1 som ::lc2 som ::right2 [aeiko]
+::s1 ii ::s2 a ::cost 0.15 ::lc1 som ::lc2 som
+::s1 y ::s2 dj ::cost 0.2 ::lc2 som
+::s1 ca ::s2 a ::cost 0.15 ::left1 /^(.*[-, ])?$/ ::lc1 som
+::s1 c ::s2 ::cost 0.25 ::left1 /^(.*[-, ])?$/ ::lc1 som
+::s1 x ::s2 h ::cost 0.25 ::lc1 som
+::s1 x ::s2 h ::cost 0.05 ::lc1 som ::left1 /^(.* )?$/ ::right1 [aeiou]
+::s1 x ::s2 h ::cost 0.1 ::lc1 som ::left1 /[aeiou]$/
+::s1 b ::s2 p ::cost 0.1 ::lc1 som
+::s1 majm ::s2 mahm ::cost 0.1 ::lc1 som
+::s1 chalim ::s2 halim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 chalim ::s2 jalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 chalim ::s2 kalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 halim ::s2 jalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 halim ::s2 kalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 jalim ::s2 kalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 dh ::s2 r ::cost 0.25 ::lc1 som ::lc2 som ::left1 /[aeiou]$/
+::s1 j ::s2 ch ::cost 0.25 ::lc1 som ::lc2 som
+::s1 j ::s2 kh ::cost 0.25 ::lc1 som ::lc2 som
+::s1 ch ::s2 sh ::cost 0.2 ::lc1 som ::lc2 som
+
+# French
+::s1 aud ::s2 o ::cost 0.3 ::right1 [-,$ ] ::lc1 eng, fra
+::s1 aux ::s2 o ::cost 0.05 ::right1 [-,$ ]
+::s1 eaux ::s2 o ::cost 0.05 ::right1 [-,$ ]
+::s1 eux ::s2 o ::cost 0.05 ::right1 [-,$ ]
+::s1 eux ::s2 e ::cost 0.15 ::right1 [-,$ ]
+
+::s1 - ::s2 " " ::cost 0.1
+::s1 : ::s2 , ::cost 0.1 ::lc1 amh
+
+# mini dictionary Amharic-English
+::s1 dabube ::s2 south ::cost 0 ::lc1 amh ::lc2 eng
+::s1 daseete ::s2 island ::cost 0 ::lc1 amh ::lc2 eng
+::s1 daseetoche ::s2 islands ::cost 0 ::lc1 amh ::lc2 eng
+::s1 kaaweneti ::s2 county ::cost 0 ::lc1 amh ::lc2 eng
+::s1 katamaa ::s2 city ::cost 0 ::lc1 amh ::lc2 eng
+::s1 kelele ::s2 region ::cost 0 ::lc1 amh ::lc2 eng
+::s1 meseraaqe ::s2 east ::cost 0 ::lc1 amh ::lc2 eng
+::s1 sameene ::s2 north ::cost 0 ::lc1 amh ::lc2 eng
+::s1 setaadiyame ::s2 stadium ::cost 0 ::lc1 amh ::lc2 eng
+::s1 waneze ::s2 river ::cost 0 ::lc1 amh ::lc2 eng
+
+# mini dictionary Arabic-English
+::s1 " " ::s2 " of " ::cost 0 ::lc1 ara ::lc2 eng
+::s1 " alawl" ::s2 " i" ::cost 0 ::lc1 ara ::lc2 eng ::right2 [-,$ ]
+
+# mini dictionary Bengali-English
+::s1 anychala ::s2 zone ::cost 0 ::lc1 ben ::lc2 eng
+::s1 pradesha ::s2 province ::cost 0 ::lc1 ben ::lc2 eng
+::s1 saamraajya ::s2 empire ::cost 0 ::lc1 ben ::lc2 eng
+::s1 upajelaa ::s2 upazila ::cost 0 ::lc1 ben ::lc2 eng
+::s1 uttara ::s2 north ::cost 0 ::lc1 ben ::lc2 eng
+::s1 "dya " ::s2 "the " ::left1 /^(.*[-, ])?$/ ::cost 0.2 ::lc1 ben ::lc2 eng
+::s1 " aba " ::s2 " of " ::cost 0 ::lc1 ben ::lc2 eng
+
+# mini dictionary Russian-English
+::s1 akademiya ::s2 academy ::cost 0 ::lc1 rus ::lc2 eng
+::s1 eparkhiya ::s2 diocese ::cost 0 ::lc1 rus ::lc2 eng
+::s1 gorod ::s2 city ::cost 0 ::lc1 rus ::lc2 eng
+::s1 gosudarstvennyi ::s2 state ::cost 0 ::lc1 rus ::lc2 eng
+::s1 gubernator ::s2 governor ::cost 0 ::lc1 rus ::lc2 eng
+::s1 guberniya ::s2 governate ::cost 0 ::lc1 rus ::lc2 eng
+::s1 imperator ::s2 emperor ::cost 0 ::lc1 rus ::lc2 eng
+::s1 komitet ::s2 committee ::cost 0 ::lc1 rus ::lc2 eng
+::s1 korolevstvo ::s2 kingdom ::cost 0 ::lc1 rus ::lc2 eng
+::s1 koroli ::s2 king ::cost 0 ::lc1 rus ::lc2 eng
+::s1 mezhdunarodnaya ::s2 international ::cost 0 ::lc1 rus ::lc2 eng
+::s1 natsionalnyi ::s2 national ::cost 0 ::lc1 rus ::lc2 eng
+::s1 novyi ::s2 new ::cost 0 ::lc1 rus ::lc2 eng
+::s1 oblast ::s2 province ::cost 0 ::lc1 rus ::lc2 eng
+::s1 oblast ::s2 region ::cost 0 ::lc1 rus ::lc2 eng
+::s1 obshchestvo ::s2 society ::cost 0 ::lc1 rus ::lc2 eng
+::s1 okrug ::s2 district ::cost 0 ::lc1 rus ::lc2 eng
+::s1 okrug ::s2 region ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ostrova ::s2 island ::cost 0 ::lc1 rus ::lc2 eng
+::s1 partiya ::s2 party ::cost 0 ::lc1 rus ::lc2 eng
+::s1 raion ::s2 district ::cost 0 ::lc1 rus ::lc2 eng
+::s1 respublika ::s2 republic ::cost 0 ::lc1 rus ::lc2 eng
+::s1 respublik ::s2 republic ::cost 0 ::lc1 rus ::lc2 eng
+::s1 sbornaya ::s2 team ::cost 0 ::lc1 rus ::lc2 eng
+::s1 severnaya ::s2 north ::cost 0 ::lc1 rus ::lc2 eng
+::s1 sovet council ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 alliance ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 association ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 league ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 union ::cost 0 ::lc1 rus ::lc2 eng
+::s1 svyataya ::s2 saint ::cost 0 ::lc1 rus ::lc2 eng
+::s1 svobodnyi ::s2 free ::cost 0 ::lc1 rus ::lc2 eng
+::s1 tserkov ::s2 church ::cost 0 ::lc1 rus ::lc2 eng
+::s1 uezd ::s2 county ::cost 0 ::lc1 rus ::lc2 eng
+::s1 universitet ::s2 university ::cost 0 ::lc1 rus ::lc2 eng
+::s1 vostochnaya ::s2 east ::cost 0 ::lc1 rus ::lc2 eng
+::s1 vostochnaya ::s2 eastern ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnaya ::s2 south ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnaya ::s2 southern ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnoi ::s2 south ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnoi ::s2 southern ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnyi ::s2 south ::cost 0 ::lc1 rus ::lc2 eng
+# often dropped in Russian name
+::s1 ::s2 county ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 island ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 pope ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 river ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 "the " ::cost 0 ::lc1 rus ::lc2 eng ::left2 /^(.*[- ])?$/
+::s1 " " ::s2 " of " ::cost 0 ::lc1 rus ::lc2 eng
+
+
+# mini dictionary Uyghur-English
+::s1 aptonom ::s2 automomous ::cost 0 ::lc1 uig ::lc2 eng
+::s1 aralliri ::s2 islands ::cost 0 ::lc1 uig ::lc2 eng
+::s1 aralliri ::s2           ::cost 0 ::lc1 uig ::lc2 eng
+::s1 arili ::s2 island ::cost 0 ::lc1 uig ::lc2 eng
+::s1 arili ::s2          ::cost 0 ::lc1 uig ::lc2 eng
+::s1 nahiyisi ::s2 county ::cost 0 ::lc1 uig ::lc2 eng
+::s1 oelkisi ::s2 province ::cost 0 ::lc1 uig ::lc2 eng
+::s1 oelkisi ::s2             ::cost 0 ::lc1 uig ::lc2 eng
+::s1 ottura ::s2 central ::cost 0 ::lc1 uig ::lc2 eng
+::s1 rayoni ::s2 region ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shehiri ::s2 city ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shehiri ::s2       ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shitati ::s2 state ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shitati ::s2         ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shtati ::s2 state ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shtati ::s2         ::cost 0 ::lc1 uig ::lc2 eng
+::s1 uniwersiteti ::s2 university ::cost 0 ::lc1 uig ::lc2 eng
+::s1 yengi ::s2 new ::cost 0 ::lc1 uig ::lc2 eng
+

uroman/lib/JSON.pm

@@ -0,0 +1,2317 @@
+package JSON;
+
+
+use strict;
+use Carp ();
+use base qw(Exporter);
+@JSON::EXPORT = qw(from_json to_json jsonToObj objToJson encode_json decode_json);
+
+BEGIN {
+    $JSON::VERSION = '2.90';
+    $JSON::DEBUG   = 0 unless (defined $JSON::DEBUG);
+    $JSON::DEBUG   = $ENV{ PERL_JSON_DEBUG } if exists $ENV{ PERL_JSON_DEBUG };
+}
+
+my $Module_XS  = 'JSON::XS';
+my $Module_PP  = 'JSON::PP';
+my $Module_bp  = 'JSON::backportPP'; # included in JSON distribution
+my $PP_Version = '2.27203';
+my $XS_Version = '2.34';
+
+
+# XS and PP common methods
+
+my @PublicMethods = qw/
+    ascii latin1 utf8 pretty indent space_before space_after relaxed canonical allow_nonref 
+    allow_blessed convert_blessed filter_json_object filter_json_single_key_object 
+    shrink max_depth max_size encode decode decode_prefix allow_unknown
+/;
+
+my @Properties = qw/
+    ascii latin1 utf8 indent space_before space_after relaxed canonical allow_nonref
+    allow_blessed convert_blessed shrink max_depth max_size allow_unknown
+/;
+
+my @XSOnlyMethods = qw/allow_tags/; # Currently nothing
+
+my @PPOnlyMethods = qw/
+    indent_length sort_by
+    allow_singlequote allow_bignum loose allow_barekey escape_slash as_nonblessed
+/; # JSON::PP specific
+
+
+# used in _load_xs and _load_pp ($INSTALL_ONLY is not used currently)
+my $_INSTALL_DONT_DIE  = 1; # When _load_xs fails to load XS, don't die.
+my $_INSTALL_ONLY      = 2; # Don't call _set_methods()
+my $_ALLOW_UNSUPPORTED = 0;
+my $_UNIV_CONV_BLESSED = 0;
+my $_USSING_bpPP       = 0;
+
+
+# Check the environment variable to decide worker module. 
+
+unless ($JSON::Backend) {
+    $JSON::DEBUG and  Carp::carp("Check used worker module...");
+
+    my $backend = exists $ENV{PERL_JSON_BACKEND} ? $ENV{PERL_JSON_BACKEND} : 1;
+
+    if ($backend eq '1' or $backend =~ /JSON::XS\s*,\s*JSON::PP/) {
+        _load_xs($_INSTALL_DONT_DIE) or _load_pp();
+    }
+    elsif ($backend eq '0' or $backend eq 'JSON::PP') {
+        _load_pp();
+    }
+    elsif ($backend eq '2' or $backend eq 'JSON::XS') {
+        _load_xs();
+    }
+    elsif ($backend eq 'JSON::backportPP') {
+        $_USSING_bpPP = 1;
+        _load_pp();
+    }
+    else {
+        Carp::croak "The value of environmental variable 'PERL_JSON_BACKEND' is invalid.";
+    }
+}
+
+
+sub import {
+    my $pkg = shift;
+    my @what_to_export;
+    my $no_export;
+
+    for my $tag (@_) {
+        if ($tag eq '-support_by_pp') {
+            if (!$_ALLOW_UNSUPPORTED++) {
+                JSON::Backend::XS
+                    ->support_by_pp(@PPOnlyMethods) if ($JSON::Backend eq $Module_XS);
+            }
+            next;
+        }
+        elsif ($tag eq '-no_export') {
+            $no_export++, next;
+        }
+        elsif ( $tag eq '-convert_blessed_universally' ) {
+            eval q|
+                require B;
+                *UNIVERSAL::TO_JSON = sub {
+                    my $b_obj = B::svref_2object( $_[0] );
+                    return    $b_obj->isa('B::HV') ? { %{ $_[0] } }
+                            : $b_obj->isa('B::AV') ? [ @{ $_[0] } ]
+                            : undef
+                            ;
+                }
+            | if ( !$_UNIV_CONV_BLESSED++ );
+            next;
+        }
+        push @what_to_export, $tag;
+    }
+
+    return if ($no_export);
+
+    __PACKAGE__->export_to_level(1, $pkg, @what_to_export);
+}
+
+
+# OBSOLETED
+
+sub jsonToObj {
+    my $alternative = 'from_json';
+    if (defined $_[0] and UNIVERSAL::isa($_[0], 'JSON')) {
+        shift @_; $alternative = 'decode';
+    }
+    Carp::carp "'jsonToObj' will be obsoleted. Please use '$alternative' instead.";
+    return JSON::from_json(@_);
+};
+
+sub objToJson {
+    my $alternative = 'to_json';
+    if (defined $_[0] and UNIVERSAL::isa($_[0], 'JSON')) {
+        shift @_; $alternative = 'encode';
+    }
+    Carp::carp "'objToJson' will be obsoleted. Please use '$alternative' instead.";
+    JSON::to_json(@_);
+};
+
+
+# INTERFACES
+
+sub to_json ($@) {
+    if (
+        ref($_[0]) eq 'JSON'
+        or (@_ > 2 and $_[0] eq 'JSON')
+    ) {
+        Carp::croak "to_json should not be called as a method.";
+    }
+    my $json = JSON->new;
+
+    if (@_ == 2 and ref $_[1] eq 'HASH') {
+        my $opt  = $_[1];
+        for my $method (keys %$opt) {
+            $json->$method( $opt->{$method} );
+        }
+    }
+
+    $json->encode($_[0]);
+}
+
+
+sub from_json ($@) {
+    if ( ref($_[0]) eq 'JSON' or $_[0] eq 'JSON' ) {
+        Carp::croak "from_json should not be called as a method.";
+    }
+    my $json = JSON->new;
+
+    if (@_ == 2 and ref $_[1] eq 'HASH') {
+        my $opt  = $_[1];
+        for my $method (keys %$opt) {
+            $json->$method( $opt->{$method} );
+        }
+    }
+
+    return $json->decode( $_[0] );
+}
+
+
+
+sub true  { $JSON::true  }
+
+sub false { $JSON::false }
+
+sub null  { undef; }
+
+
+sub require_xs_version { $XS_Version; }
+
+sub backend {
+    my $proto = shift;
+    $JSON::Backend;
+}
+
+#*module = *backend;
+
+
+sub is_xs {
+    return $_[0]->backend eq $Module_XS;
+}
+
+
+sub is_pp {
+    return not $_[0]->is_xs;
+}
+
+
+sub pureperl_only_methods { @PPOnlyMethods; }
+
+
+sub property {
+    my ($self, $name, $value) = @_;
+
+    if (@_ == 1) {
+        my %props;
+        for $name (@Properties) {
+            my $method = 'get_' . $name;
+            if ($name eq 'max_size') {
+                my $value = $self->$method();
+                $props{$name} = $value == 1 ? 0 : $value;
+                next;
+            }
+            $props{$name} = $self->$method();
+        }
+        return \%props;
+    }
+    elsif (@_ > 3) {
+        Carp::croak('property() can take only the option within 2 arguments.');
+    }
+    elsif (@_ == 2) {
+        if ( my $method = $self->can('get_' . $name) ) {
+            if ($name eq 'max_size') {
+                my $value = $self->$method();
+                return $value == 1 ? 0 : $value;
+            }
+            $self->$method();
+        }
+    }
+    else {
+        $self->$name($value);
+    }
+
+}
+
+
+
+# INTERNAL
+
+sub _load_xs {
+    my $opt = shift;
+
+    $JSON::DEBUG and Carp::carp "Load $Module_XS.";
+
+    # if called after install module, overload is disable.... why?
+    JSON::Boolean::_overrride_overload($Module_XS);
+    JSON::Boolean::_overrride_overload($Module_PP);
+
+    eval qq|
+        use $Module_XS $XS_Version ();
+    |;
+
+    if ($@) {
+        if (defined $opt and $opt & $_INSTALL_DONT_DIE) {
+            $JSON::DEBUG and Carp::carp "Can't load $Module_XS...($@)";
+            return 0;
+        }
+        Carp::croak $@;
+    }
+
+    unless (defined $opt and $opt & $_INSTALL_ONLY) {
+        _set_module( $JSON::Backend = $Module_XS );
+        my $data = join("", <DATA>); # this code is from Jcode 2.xx.
+        close(DATA);
+        eval $data;
+        JSON::Backend::XS->init;
+    }
+
+    return 1;
+};
+
+
+sub _load_pp {
+    my $opt = shift;
+    my $backend = $_USSING_bpPP ? $Module_bp : $Module_PP;
+
+    $JSON::DEBUG and Carp::carp "Load $backend.";
+
+    # if called after install module, overload is disable.... why?
+    JSON::Boolean::_overrride_overload($Module_XS);
+    JSON::Boolean::_overrride_overload($backend);
+
+    if ( $_USSING_bpPP ) {
+        eval qq| require $backend |;
+    }
+    else {
+        eval qq| use $backend $PP_Version () |;
+    }
+
+    if ($@) {
+        if ( $backend eq $Module_PP ) {
+            $JSON::DEBUG and Carp::carp "Can't load $Module_PP ($@), so try to load $Module_bp";
+            $_USSING_bpPP++;
+            $backend = $Module_bp;
+            JSON::Boolean::_overrride_overload($backend);
+            local $^W; # if PP installed but invalid version, backportPP redefines methods.
+            eval qq| require $Module_bp |;
+        }
+        Carp::croak $@ if $@;
+    }
+
+    unless (defined $opt and $opt & $_INSTALL_ONLY) {
+        _set_module( $JSON::Backend = $Module_PP ); # even if backportPP, set $Backend with 'JSON::PP'
+        JSON::Backend::PP->init;
+    }
+};
+
+
+sub _set_module {
+    return if defined $JSON::true;
+
+    my $module = shift;
+
+    local $^W;
+    no strict qw(refs);
+
+    $JSON::true  = ${"$module\::true"};
+    $JSON::false = ${"$module\::false"};
+
+    push @JSON::ISA, $module;
+    if ( JSON->is_xs and JSON->backend->VERSION < 3 ) {
+        eval 'package JSON::PP::Boolean';
+        push @{"$module\::Boolean::ISA"}, qw(JSON::PP::Boolean);
+    }
+
+    *{"JSON::is_bool"} = \&{"$module\::is_bool"};
+
+    for my $method ($module eq $Module_XS ? @PPOnlyMethods : @XSOnlyMethods) {
+        *{"JSON::$method"} = sub {
+            Carp::carp("$method is not supported in $module.");
+            $_[0];
+        };
+    }
+
+    return 1;
+}
+
+
+
+#
+# JSON Boolean
+#
+
+package JSON::Boolean;
+
+my %Installed;
+
+sub _overrride_overload {
+    return; # this function is currently disable.
+    return if ($Installed{ $_[0] }++);
+
+    my $boolean = $_[0] . '::Boolean';
+
+    eval sprintf(q|
+        package %s;
+        use overload (
+            '""' => sub { ${$_[0]} == 1 ? 'true' : 'false' },
+            'eq' => sub {
+                my ($obj, $op) = ref ($_[0]) ? ($_[0], $_[1]) : ($_[1], $_[0]);
+                if ($op eq 'true' or $op eq 'false') {
+                    return "$obj" eq 'true' ? 'true' eq $op : 'false' eq $op;
+                }
+                else {
+                    return $obj ? 1 == $op : 0 == $op;
+                }
+            },
+        );
+    |, $boolean);
+
+    if ($@) { Carp::croak $@; }
+
+    if ( exists $INC{'JSON/XS.pm'} and $boolean eq 'JSON::XS::Boolean' ) {
+        local $^W;
+        my $true  = do { bless \(my $dummy = 1), $boolean };
+        my $false = do { bless \(my $dummy = 0), $boolean };
+        *JSON::XS::true  = sub () { $true };
+        *JSON::XS::false = sub () { $false };
+    }
+    elsif ( exists $INC{'JSON/PP.pm'} and $boolean eq 'JSON::PP::Boolean' ) {
+        local $^W;
+        my $true  = do { bless \(my $dummy = 1), $boolean };
+        my $false = do { bless \(my $dummy = 0), $boolean };
+        *JSON::PP::true  = sub { $true };
+        *JSON::PP::false = sub { $false };
+    }
+
+    return 1;
+}
+
+
+#
+# Helper classes for Backend Module (PP)
+#
+
+package JSON::Backend::PP;
+
+sub init {
+    local $^W;
+    no strict qw(refs); # this routine may be called after JSON::Backend::XS init was called.
+    *{"JSON::decode_json"} = \&{"JSON::PP::decode_json"};
+    *{"JSON::encode_json"} = \&{"JSON::PP::encode_json"};
+    *{"JSON::PP::is_xs"}  = sub { 0 };
+    *{"JSON::PP::is_pp"}  = sub { 1 };
+    return 1;
+}
+
+#
+# To save memory, the below lines are read only when XS backend is used.
+#
+
+package JSON;
+
+1;
+__DATA__
+
+
+#
+# Helper classes for Backend Module (XS)
+#
+
+package JSON::Backend::XS;
+
+use constant INDENT_LENGTH_FLAG => 15 << 12;
+
+use constant UNSUPPORTED_ENCODE_FLAG => {
+    ESCAPE_SLASH      => 0x00000010,
+    ALLOW_BIGNUM      => 0x00000020,
+    AS_NONBLESSED     => 0x00000040,
+    EXPANDED          => 0x10000000, # for developer's
+};
+
+use constant UNSUPPORTED_DECODE_FLAG => {
+    LOOSE             => 0x00000001,
+    ALLOW_BIGNUM      => 0x00000002,
+    ALLOW_BAREKEY     => 0x00000004,
+    ALLOW_SINGLEQUOTE => 0x00000008,
+    EXPANDED          => 0x20000000, # for developer's
+};
+
+
+sub init {
+    local $^W;
+    no strict qw(refs);
+    *{"JSON::decode_json"} = \&{"JSON::XS::decode_json"};
+    *{"JSON::encode_json"} = \&{"JSON::XS::encode_json"};
+    *{"JSON::XS::is_xs"}  = sub { 1 };
+    *{"JSON::XS::is_pp"}  = sub { 0 };
+    return 1;
+}
+
+
+sub support_by_pp {
+    my ($class, @methods) = @_;
+
+    local $^W;
+    no strict qw(refs);
+
+    my $JSON_XS_encode_orignal     = \&JSON::XS::encode;
+    my $JSON_XS_decode_orignal     = \&JSON::XS::decode;
+    my $JSON_XS_incr_parse_orignal = \&JSON::XS::incr_parse;
+
+    *JSON::XS::decode     = \&JSON::Backend::XS::Supportable::_decode;
+    *JSON::XS::encode     = \&JSON::Backend::XS::Supportable::_encode;
+    *JSON::XS::incr_parse = \&JSON::Backend::XS::Supportable::_incr_parse;
+
+    *{JSON::XS::_original_decode}     = $JSON_XS_decode_orignal;
+    *{JSON::XS::_original_encode}     = $JSON_XS_encode_orignal;
+    *{JSON::XS::_original_incr_parse} = $JSON_XS_incr_parse_orignal;
+
+    push @JSON::Backend::XS::Supportable::ISA, 'JSON';
+
+    my $pkg = 'JSON::Backend::XS::Supportable';
+
+    *{JSON::new} = sub {
+        my $proto = JSON::XS->new; $$proto = 0;
+        bless  $proto, $pkg;
+    };
+
+
+    for my $method (@methods) {
+        my $flag = uc($method);
+        my $type |= (UNSUPPORTED_ENCODE_FLAG->{$flag} || 0);
+           $type |= (UNSUPPORTED_DECODE_FLAG->{$flag} || 0);
+
+        next unless($type);
+
+        $pkg->_make_unsupported_method($method => $type);
+    }
+
+#    push @{"JSON::XS::Boolean::ISA"}, qw(JSON::PP::Boolean);
+#    push @{"JSON::PP::Boolean::ISA"}, qw(JSON::Boolean);
+
+    $JSON::DEBUG and Carp::carp("set -support_by_pp mode.");
+
+    return 1;
+}
+
+
+
+
+#
+# Helper classes for XS
+#
+
+package JSON::Backend::XS::Supportable;
+
+$Carp::Internal{'JSON::Backend::XS::Supportable'} = 1;
+
+sub _make_unsupported_method {
+    my ($pkg, $method, $type) = @_;
+
+    local $^W;
+    no strict qw(refs);
+
+    *{"$pkg\::$method"} = sub {
+        local $^W;
+        if (defined $_[1] ? $_[1] : 1) {
+            ${$_[0]} |= $type;
+        }
+        else {
+            ${$_[0]} &= ~$type;
+        }
+        $_[0];
+    };
+
+    *{"$pkg\::get_$method"} = sub {
+        ${$_[0]} & $type ? 1 : '';
+    };
+
+}
+
+
+sub _set_for_pp {
+    JSON::_load_pp( $_INSTALL_ONLY );
+
+    my $type  = shift;
+    my $pp    = JSON::PP->new;
+    my $prop = $_[0]->property;
+
+    for my $name (keys %$prop) {
+        $pp->$name( $prop->{$name} ? $prop->{$name} : 0 );
+    }
+
+    my $unsupported = $type eq 'encode' ? JSON::Backend::XS::UNSUPPORTED_ENCODE_FLAG
+                                        : JSON::Backend::XS::UNSUPPORTED_DECODE_FLAG;
+    my $flags       = ${$_[0]} || 0;
+
+    for my $name (keys %$unsupported) {
+        next if ($name eq 'EXPANDED'); # for developer's
+        my $enable = ($flags & $unsupported->{$name}) ? 1 : 0;
+        my $method = lc $name;
+        $pp->$method($enable);
+    }
+
+    $pp->indent_length( $_[0]->get_indent_length );
+
+    return $pp;
+}
+
+sub _encode { # using with PP encode
+    if (${$_[0]}) {
+        _set_for_pp('encode' => @_)->encode($_[1]);
+    }
+    else {
+        $_[0]->_original_encode( $_[1] );
+    }
+}
+
+
+sub _decode { # if unsupported-flag is set, use PP
+    if (${$_[0]}) {
+        _set_for_pp('decode' => @_)->decode($_[1]);
+    }
+    else {
+        $_[0]->_original_decode( $_[1] );
+    }
+}
+
+
+sub decode_prefix { # if unsupported-flag is set, use PP
+    _set_for_pp('decode' => @_)->decode_prefix($_[1]);
+}
+
+
+sub _incr_parse {
+    if (${$_[0]}) {
+        _set_for_pp('decode' => @_)->incr_parse($_[1]);
+    }
+    else {
+        $_[0]->_original_incr_parse( $_[1] );
+    }
+}
+
+
+sub get_indent_length {
+    ${$_[0]} << 4 >> 16;
+}
+
+
+sub indent_length {
+    my $length = $_[1];
+
+    if (!defined $length or $length > 15 or $length < 0) {
+        Carp::carp "The acceptable range of indent_length() is 0 to 15.";
+    }
+    else {
+        local $^W;
+        $length <<= 12;
+        ${$_[0]} &= ~ JSON::Backend::XS::INDENT_LENGTH_FLAG;
+        ${$_[0]} |= $length;
+        *JSON::XS::encode = \&JSON::Backend::XS::Supportable::_encode;
+    }
+
+    $_[0];
+}
+
+
+1;
+__END__
+
+=head1 NAME
+
+JSON - JSON (JavaScript Object Notation) encoder/decoder
+
+=head1 SYNOPSIS
+
+ use JSON; # imports encode_json, decode_json, to_json and from_json.
+ 
+ # simple and fast interfaces (expect/generate UTF-8)
+ 
+ $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref;
+ $perl_hash_or_arrayref  = decode_json $utf8_encoded_json_text;
+ 
+ # OO-interface
+ 
+ $json = JSON->new->allow_nonref;
+ 
+ $json_text   = $json->encode( $perl_scalar );
+ $perl_scalar = $json->decode( $json_text );
+ 
+ $pretty_printed = $json->pretty->encode( $perl_scalar ); # pretty-printing
+ 
+ # If you want to use PP only support features, call with '-support_by_pp'
+ # When XS unsupported feature is enable, using PP (de|en)code instead of XS ones.
+ 
+ use JSON -support_by_pp;
+ 
+ # option-acceptable interfaces (expect/generate UNICODE by default)
+ 
+ $json_text   = to_json( $perl_scalar, { ascii => 1, pretty => 1 } );
+ $perl_scalar = from_json( $json_text, { utf8  => 1 } );
+ 
+ # Between (en|de)code_json and (to|from)_json, if you want to write
+ # a code which communicates to an outer world (encoded in UTF-8),
+ # recommend to use (en|de)code_json.
+ 
+=head1 VERSION
+
+    2.90
+
+This version is compatible with JSON::XS B<2.34> and later.
+(Not yet compatble to JSON::XS B<3.0x>.)
+
+
+=head1 NOTE
+
+JSON::PP was earlier included in the C<JSON> distribution, but
+has since Perl 5.14 been a core module. For this reason,
+L<JSON::PP> was removed from the JSON distribution and can now
+be found also in the Perl5 repository at
+
+=over
+
+=item * L<http://perl5.git.perl.org/perl.git>
+
+=back
+
+(The newest JSON::PP version still exists in CPAN.)
+
+Instead, the C<JSON> distribution will include JSON::backportPP
+for backwards computability. JSON.pm should thus work as it did
+before.
+
+=head1 DESCRIPTION
+
+ *************************** CAUTION **************************************
+ *                                                                        *
+ * INCOMPATIBLE CHANGE (JSON::XS version 2.90)                            *
+ *                                                                        *
+ * JSON.pm had patched JSON::XS::Boolean and JSON::PP::Boolean internally *
+ * on loading time for making these modules inherit JSON::Boolean.        *
+ * But since JSON::XS v3.0 it use Types::Serialiser as boolean class.     *
+ * Then now JSON.pm breaks boolean classe overload features and           *
+ * -support_by_pp if JSON::XS v3.0 or later is installed.                 *
+ *                                                                        *
+ * JSON::true and JSON::false returned JSON::Boolean objects.             *
+ * For workaround, they return JSON::PP::Boolean objects in this version. *
+ *                                                                        *
+ *     isa_ok(JSON::true, 'JSON::PP::Boolean');                           *
+ *                                                                        *
+ * And it discards a feature:                                             *
+ *                                                                        *
+ *     ok(JSON::true eq 'true');                                          *
+ *                                                                        *
+ * In other word, JSON::PP::Boolean overload numeric only.                *
+ *                                                                        *
+ *     ok( JSON::true == 1 );                                             *
+ *                                                                        *
+ **************************************************************************
+
+ ************************** CAUTION ********************************
+ * This is 'JSON module version 2' and there are many differences  *
+ * to version 1.xx                                                 *
+ * Please check your applications using old version.              *
+ *   See to 'INCOMPATIBLE CHANGES TO OLD VERSION'                  *
+ *******************************************************************
+
+JSON (JavaScript Object Notation) is a simple data format.
+See to L<http://www.json.org/> and C<RFC4627>(L<http://www.ietf.org/rfc/rfc4627.txt>).
+
+This module converts Perl data structures to JSON and vice versa using either
+L<JSON::XS> or L<JSON::PP>.
+
+JSON::XS is the fastest and most proper JSON module on CPAN which must be
+compiled and installed in your environment.
+JSON::PP is a pure-Perl module which is bundled in this distribution and
+has a strong compatibility to JSON::XS.
+
+This module try to use JSON::XS by default and fail to it, use JSON::PP instead.
+So its features completely depend on JSON::XS or JSON::PP.
+
+See to L<BACKEND MODULE DECISION>.
+
+To distinguish the module name 'JSON' and the format type JSON,
+the former is quoted by CE<lt>E<gt> (its results vary with your using media),
+and the latter is left just as it is.
+
+Module name : C<JSON>
+
+Format type : JSON
+
+=head2 FEATURES
+
+=over
+
+=item * correct unicode handling
+
+This module (i.e. backend modules) knows how to handle Unicode, documents
+how and when it does so, and even documents what "correct" means.
+
+Even though there are limitations, this feature is available since Perl version 5.6.
+
+JSON::XS requires Perl 5.8.2 (but works correctly in 5.8.8 or later), so in older versions
+C<JSON> should call JSON::PP as the backend which can be used since Perl 5.005.
+
+With Perl 5.8.x JSON::PP works, but from 5.8.0 to 5.8.2, because of a Perl side problem,
+JSON::PP works slower in the versions. And in 5.005, the Unicode handling is not available.
+See to L<JSON::PP/UNICODE HANDLING ON PERLS> for more information.
+
+See also to L<JSON::XS/A FEW NOTES ON UNICODE AND PERL>
+and L<JSON::XS/ENCODING/CODESET_FLAG_NOTES>.
+
+
+=item * round-trip integrity
+
+When you serialise a perl data structure using only data types supported
+by JSON and Perl, the deserialised data structure is identical on the Perl
+level. (e.g. the string "2.0" doesn't suddenly become "2" just because
+it looks like a number). There I<are> minor exceptions to this, read the
+L</MAPPING> section below to learn about those.
+
+
+=item * strict checking of JSON correctness
+
+There is no guessing, no generating of illegal JSON texts by default,
+and only JSON is accepted as input by default (the latter is a security
+feature).
+
+See to L<JSON::XS/FEATURES> and L<JSON::PP/FEATURES>.
+
+=item * fast
+
+This module returns a JSON::XS object itself if available.
+Compared to other JSON modules and other serialisers such as Storable,
+JSON::XS usually compares favorably in terms of speed, too.
+
+If not available, C<JSON> returns a JSON::PP object instead of JSON::XS and
+it is very slow as pure-Perl.
+
+=item * simple to use
+
+This module has both a simple functional interface as well as an
+object oriented interface interface.
+
+=item * reasonably versatile output formats
+
+You can choose between the most compact guaranteed-single-line format possible
+(nice for simple line-based protocols), a pure-ASCII format (for when your transport
+is not 8-bit clean, still supports the whole Unicode range), or a pretty-printed
+format (for when you want to read that stuff). Or you can combine those features
+in whatever way you like.
+
+=back
+
+=head1 FUNCTIONAL INTERFACE
+
+Some documents are copied and modified from L<JSON::XS/FUNCTIONAL INTERFACE>.
+C<to_json> and C<from_json> are additional functions.
+
+=head2 encode_json
+
+    $json_text = encode_json $perl_scalar
+
+Converts the given Perl data structure to a UTF-8 encoded, binary string.
+
+This function call is functionally identical to:
+
+    $json_text = JSON->new->utf8->encode($perl_scalar)
+
+=head2 decode_json
+
+    $perl_scalar = decode_json $json_text
+
+The opposite of C<encode_json>: expects an UTF-8 (binary) string and tries
+to parse that as an UTF-8 encoded JSON text, returning the resulting
+reference.
+
+This function call is functionally identical to:
+
+    $perl_scalar = JSON->new->utf8->decode($json_text)
+
+
+=head2 to_json
+
+   $json_text = to_json($perl_scalar)
+
+Converts the given Perl data structure to a json string.
+
+This function call is functionally identical to:
+
+   $json_text = JSON->new->encode($perl_scalar)
+
+Takes a hash reference as the second.
+
+   $json_text = to_json($perl_scalar, $flag_hashref)
+
+So,
+
+   $json_text = to_json($perl_scalar, {utf8 => 1, pretty => 1})
+
+equivalent to:
+
+   $json_text = JSON->new->utf8(1)->pretty(1)->encode($perl_scalar)
+
+If you want to write a modern perl code which communicates to outer world,
+you should use C<encode_json> (supposed that JSON data are encoded in UTF-8).
+
+=head2 from_json
+
+   $perl_scalar = from_json($json_text)
+
+The opposite of C<to_json>: expects a json string and tries
+to parse it, returning the resulting reference.
+
+This function call is functionally identical to:
+
+    $perl_scalar = JSON->decode($json_text)
+
+Takes a hash reference as the second.
+
+    $perl_scalar = from_json($json_text, $flag_hashref)
+
+So,
+
+    $perl_scalar = from_json($json_text, {utf8 => 1})
+
+equivalent to:
+
+    $perl_scalar = JSON->new->utf8(1)->decode($json_text)
+
+If you want to write a modern perl code which communicates to outer world,
+you should use C<decode_json> (supposed that JSON data are encoded in UTF-8).
+
+=head2 JSON::is_bool
+
+    $is_boolean = JSON::is_bool($scalar)
+
+Returns true if the passed scalar represents either JSON::true or
+JSON::false, two constants that act like C<1> and C<0> respectively
+and are also used to represent JSON C<true> and C<false> in Perl strings.
+
+=head2 JSON::true
+
+Returns JSON true value which is blessed object.
+It C<isa> JSON::Boolean object.
+
+=head2 JSON::false
+
+Returns JSON false value which is blessed object.
+It C<isa> JSON::Boolean object.
+
+=head2 JSON::null
+
+Returns C<undef>.
+
+See L<MAPPING>, below, for more information on how JSON values are mapped to
+Perl.
+
+=head1 HOW DO I DECODE A DATA FROM OUTER AND ENCODE TO OUTER
+
+This section supposes that your perl version is 5.8 or later.
+
+If you know a JSON text from an outer world - a network, a file content, and so on,
+is encoded in UTF-8, you should use C<decode_json> or C<JSON> module object
+with C<utf8> enable. And the decoded result will contain UNICODE characters.
+
+  # from network
+  my $json        = JSON->new->utf8;
+  my $json_text   = CGI->new->param( 'json_data' );
+  my $perl_scalar = $json->decode( $json_text );
+  
+  # from file content
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  $json_text   = <$fh>;
+  $perl_scalar = decode_json( $json_text );
+
+If an outer data is not encoded in UTF-8, firstly you should C<decode> it.
+
+  use Encode;
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  my $encoding = 'cp932';
+  my $unicode_json_text = decode( $encoding, <$fh> ); # UNICODE
+  
+  # or you can write the below code.
+  #
+  # open( my $fh, "<:encoding($encoding)", 'json.data' );
+  # $unicode_json_text = <$fh>;
+
+In this case, C<$unicode_json_text> is of course UNICODE string.
+So you B<cannot> use C<decode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable or C<from_json>.
+
+  $perl_scalar = $json->utf8(0)->decode( $unicode_json_text );
+  # or
+  $perl_scalar = from_json( $unicode_json_text );
+
+Or C<encode 'utf8'> and C<decode_json>:
+
+  $perl_scalar = decode_json( encode( 'utf8', $unicode_json_text ) );
+  # this way is not efficient.
+
+And now, you want to convert your C<$perl_scalar> into JSON data and
+send it to an outer world - a network or a file content, and so on.
+
+Your data usually contains UNICODE strings and you want the converted data to be encoded
+in UTF-8, you should use C<encode_json> or C<JSON> module object with C<utf8> enable.
+
+  print encode_json( $perl_scalar ); # to a network? file? or display?
+  # or
+  print $json->utf8->encode( $perl_scalar );
+
+If C<$perl_scalar> does not contain UNICODE but C<$encoding>-encoded strings
+for some reason, then its characters are regarded as B<latin1> for perl
+(because it does not concern with your $encoding).
+You B<cannot> use C<encode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable or C<to_json>.
+Note that the resulted text is a UNICODE string but no problem to print it.
+
+  # $perl_scalar contains $encoding encoded string values
+  $unicode_json_text = $json->utf8(0)->encode( $perl_scalar );
+  # or 
+  $unicode_json_text = to_json( $perl_scalar );
+  # $unicode_json_text consists of characters less than 0x100
+  print $unicode_json_text;
+
+Or C<decode $encoding> all string values and C<encode_json>:
+
+  $perl_scalar->{ foo } = decode( $encoding, $perl_scalar->{ foo } );
+  # ... do it to each string values, then encode_json
+  $json_text = encode_json( $perl_scalar );
+
+This method is a proper way but probably not efficient.
+
+See to L<Encode>, L<perluniintro>.
+
+
+=head1 COMMON OBJECT-ORIENTED INTERFACE
+
+=head2 new
+
+    $json = JSON->new
+
+Returns a new C<JSON> object inherited from either JSON::XS or JSON::PP
+that can be used to de/encode JSON strings.
+
+All boolean flags described below are by default I<disabled>.
+
+The mutators for flags all return the JSON object again and thus calls can
+be chained:
+
+   my $json = JSON->new->utf8->space_after->encode({a => [1,2]})
+   => {"a": [1, 2]}
+
+=head2 ascii
+
+    $json = $json->ascii([$enable])
+    
+    $enabled = $json->get_ascii
+
+If $enable is true (or missing), then the encode method will not generate characters outside
+the code range 0..127. Any Unicode characters outside that range will be escaped using either
+a single \uXXXX or a double \uHHHH\uLLLLL escape sequence, as per RFC4627.
+
+If $enable is false, then the encode method will not escape Unicode characters unless
+required by the JSON syntax or other flags. This results in a faster and more compact format.
+
+This feature depends on the used Perl version and environment.
+
+See to L<JSON::PP/UNICODE HANDLING ON PERLS> if the backend is PP.
+
+  JSON->new->ascii(1)->encode([chr 0x10401])
+  => ["\ud801\udc01"]
+
+=head2 latin1
+
+    $json = $json->latin1([$enable])
+    
+    $enabled = $json->get_latin1
+
+If $enable is true (or missing), then the encode method will encode the resulting JSON
+text as latin1 (or iso-8859-1), escaping any characters outside the code range 0..255.
+
+If $enable is false, then the encode method will not escape Unicode characters
+unless required by the JSON syntax or other flags.
+
+  JSON->new->latin1->encode (["\x{89}\x{abc}"]
+  => ["\x{89}\\u0abc"]    # (perl syntax, U+abc escaped, U+89 not)
+
+=head2 utf8
+
+    $json = $json->utf8([$enable])
+    
+    $enabled = $json->get_utf8
+
+If $enable is true (or missing), then the encode method will encode the JSON result
+into UTF-8, as required by many protocols, while the decode method expects to be handled
+an UTF-8-encoded string. Please note that UTF-8-encoded strings do not contain any
+characters outside the range 0..255, they are thus useful for bytewise/binary I/O.
+
+In future versions, enabling this option might enable autodetection of the UTF-16 and UTF-32
+encoding families, as described in RFC4627.
+
+If $enable is false, then the encode method will return the JSON string as a (non-encoded)
+Unicode string, while decode expects thus a Unicode string. Any decoding or encoding
+(e.g. to UTF-8 or UTF-16) needs to be done yourself, e.g. using the Encode module.
+
+
+Example, output UTF-16BE-encoded JSON:
+
+  use Encode;
+  $jsontext = encode "UTF-16BE", JSON::XS->new->encode ($object);
+
+Example, decode UTF-32LE-encoded JSON:
+
+  use Encode;
+  $object = JSON::XS->new->decode (decode "UTF-32LE", $jsontext);
+
+See to L<JSON::PP/UNICODE HANDLING ON PERLS> if the backend is PP.
+
+
+=head2 pretty
+
+    $json = $json->pretty([$enable])
+
+This enables (or disables) all of the C<indent>, C<space_before> and
+C<space_after> (and in the future possibly more) flags in one call to
+generate the most readable (or most compact) form possible.
+
+Equivalent to:
+
+   $json->indent->space_before->space_after
+
+The indent space length is three and JSON::XS cannot change the indent
+space length.
+
+=head2 indent
+
+    $json = $json->indent([$enable])
+    
+    $enabled = $json->get_indent
+
+If C<$enable> is true (or missing), then the C<encode> method will use a multiline
+format as output, putting every array member or object/hash key-value pair
+into its own line, identifying them properly.
+
+If C<$enable> is false, no newlines or indenting will be produced, and the
+resulting JSON text is guaranteed not to contain any C<newlines>.
+
+This setting has no effect when decoding JSON texts.
+
+The indent space length is three.
+With JSON::PP, you can also access C<indent_length> to change indent space length.
+
+
+=head2 space_before
+
+    $json = $json->space_before([$enable])
+    
+    $enabled = $json->get_space_before
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space before the C<:> separating keys from values in JSON objects.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before enabled, space_after and indent disabled:
+
+   {"key" :"value"}
+
+
+=head2 space_after
+
+    $json = $json->space_after([$enable])
+    
+    $enabled = $json->get_space_after
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space after the C<:> separating keys from values in JSON objects
+and extra whitespace after the C<,> separating key-value pairs and array
+members.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before and indent disabled, space_after enabled:
+
+   {"key": "value"}
+
+
+=head2 relaxed
+
+    $json = $json->relaxed([$enable])
+    
+    $enabled = $json->get_relaxed
+
+If C<$enable> is true (or missing), then C<decode> will accept some
+extensions to normal JSON syntax (see below). C<encode> will not be
+affected in anyway. I<Be aware that this option makes you accept invalid
+JSON texts as if they were valid!>. I suggest only to use this option to
+parse application-specific files written by humans (configuration files,
+resource files etc.)
+
+If C<$enable> is false (the default), then C<decode> will only accept
+valid JSON texts.
+
+Currently accepted extensions are:
+
+=over 4
+
+=item * list items can have an end-comma
+
+JSON I<separates> array elements and key-value pairs with commas. This
+can be annoying if you write JSON texts manually and want to be able to
+quickly append elements, so this extension accepts comma at the end of
+such items not just between them:
+
+   [
+      1,
+      2, <- this comma not normally allowed
+   ]
+   {
+      "k1": "v1",
+      "k2": "v2", <- this comma not normally allowed
+   }
+
+=item * shell-style '#'-comments
+
+Whenever JSON allows whitespace, shell-style comments are additionally
+allowed. They are terminated by the first carriage-return or line-feed
+character, after which more white-space and comments are allowed.
+
+  [
+     1, # this comment not allowed in JSON
+        # neither this one...
+  ]
+
+=back
+
+
+=head2 canonical
+
+    $json = $json->canonical([$enable])
+    
+    $enabled = $json->get_canonical
+
+If C<$enable> is true (or missing), then the C<encode> method will output JSON objects
+by sorting their keys. This is adding a comparatively high overhead.
+
+If C<$enable> is false, then the C<encode> method will output key-value
+pairs in the order Perl stores them (which will likely change between runs
+of the same script).
+
+This option is useful if you want the same data structure to be encoded as
+the same JSON text (given the same overall settings). If it is disabled,
+the same hash might be encoded differently even if contains the same data,
+as key-value pairs have no inherent ordering in Perl.
+
+This setting has no effect when decoding JSON texts.
+
+=head2 allow_nonref
+
+    $json = $json->allow_nonref([$enable])
+    
+    $enabled = $json->get_allow_nonref
+
+If C<$enable> is true (or missing), then the C<encode> method can convert a
+non-reference into its corresponding string, number or null JSON value,
+which is an extension to RFC4627. Likewise, C<decode> will accept those JSON
+values instead of croaking.
+
+If C<$enable> is false, then the C<encode> method will croak if it isn't
+passed an arrayref or hashref, as JSON texts must either be an object
+or array. Likewise, C<decode> will croak if given something that is not a
+JSON object or array.
+
+   JSON->new->allow_nonref->encode ("Hello, World!")
+   => "Hello, World!"
+
+=head2 allow_unknown
+
+    $json = $json->allow_unknown ([$enable])
+    
+    $enabled = $json->get_allow_unknown
+
+If $enable is true (or missing), then "encode" will *not* throw an
+exception when it encounters values it cannot represent in JSON (for
+example, filehandles) but instead will encode a JSON "null" value.
+Note that blessed objects are not included here and are handled
+separately by c<allow_nonref>.
+
+If $enable is false (the default), then "encode" will throw an
+exception when it encounters anything it cannot encode as JSON.
+
+This option does not affect "decode" in any way, and it is
+recommended to leave it off unless you know your communications
+partner.
+
+=head2 allow_blessed
+
+    $json = $json->allow_blessed([$enable])
+    
+    $enabled = $json->get_allow_blessed
+
+If C<$enable> is true (or missing), then the C<encode> method will not
+barf when it encounters a blessed reference. Instead, the value of the
+B<convert_blessed> option will decide whether C<null> (C<convert_blessed>
+disabled or no C<TO_JSON> method found) or a representation of the
+object (C<convert_blessed> enabled and C<TO_JSON> method found) is being
+encoded. Has no effect on C<decode>.
+
+If C<$enable> is false (the default), then C<encode> will throw an
+exception when it encounters a blessed object.
+
+
+=head2 convert_blessed
+
+    $json = $json->convert_blessed([$enable])
+    
+    $enabled = $json->get_convert_blessed
+
+If C<$enable> is true (or missing), then C<encode>, upon encountering a
+blessed object, will check for the availability of the C<TO_JSON> method
+on the object's class. If found, it will be called in scalar context
+and the resulting scalar will be encoded instead of the object. If no
+C<TO_JSON> method is found, the value of C<allow_blessed> will decide what
+to do.
+
+The C<TO_JSON> method may safely call die if it wants. If C<TO_JSON>
+returns other blessed objects, those will be handled in the same
+way. C<TO_JSON> must take care of not causing an endless recursion cycle
+(== crash) in this case. The name of C<TO_JSON> was chosen because other
+methods called by the Perl core (== not by the user of the object) are
+usually in upper case letters and to avoid collisions with the C<to_json>
+function or method.
+
+This setting does not yet influence C<decode> in any way.
+
+If C<$enable> is false, then the C<allow_blessed> setting will decide what
+to do when a blessed object is found.
+
+=over
+
+=item convert_blessed_universally mode
+
+If use C<JSON> with C<-convert_blessed_universally>, the C<UNIVERSAL::TO_JSON>
+subroutine is defined as the below code:
+
+   *UNIVERSAL::TO_JSON = sub {
+       my $b_obj = B::svref_2object( $_[0] );
+       return    $b_obj->isa('B::HV') ? { %{ $_[0] } }
+               : $b_obj->isa('B::AV') ? [ @{ $_[0] } ]
+               : undef
+               ;
+   }
+
+This will cause that C<encode> method converts simple blessed objects into
+JSON objects as non-blessed object.
+
+   JSON -convert_blessed_universally;
+   $json->allow_blessed->convert_blessed->encode( $blessed_object )
+
+This feature is experimental and may be removed in the future.
+
+=back
+
+=head2 filter_json_object
+
+    $json = $json->filter_json_object([$coderef])
+
+When C<$coderef> is specified, it will be called from C<decode> each
+time it decodes a JSON object. The only argument passed to the coderef
+is a reference to the newly-created hash. If the code references returns
+a single scalar (which need not be a reference), this value
+(i.e. a copy of that scalar to avoid aliasing) is inserted into the
+deserialised data structure. If it returns an empty list
+(NOTE: I<not> C<undef>, which is a valid scalar), the original deserialised
+hash will be inserted. This setting can slow down decoding considerably.
+
+When C<$coderef> is omitted or undefined, any existing callback will
+be removed and C<decode> will not change the deserialised hash in any
+way.
+
+Example, convert all JSON objects into the integer 5:
+
+   my $js = JSON->new->filter_json_object (sub { 5 });
+   # returns [5]
+   $js->decode ('[{}]'); # the given subroutine takes a hash reference.
+   # throw an exception because allow_nonref is not enabled
+   # so a lone 5 is not allowed.
+   $js->decode ('{"a":1, "b":2}');
+
+
+=head2 filter_json_single_key_object
+
+    $json = $json->filter_json_single_key_object($key [=> $coderef])
+
+Works remotely similar to C<filter_json_object>, but is only called for
+JSON objects having a single key named C<$key>.
+
+This C<$coderef> is called before the one specified via
+C<filter_json_object>, if any. It gets passed the single value in the JSON
+object. If it returns a single value, it will be inserted into the data
+structure. If it returns nothing (not even C<undef> but the empty list),
+the callback from C<filter_json_object> will be called next, as if no
+single-key callback were specified.
+
+If C<$coderef> is omitted or undefined, the corresponding callback will be
+disabled. There can only ever be one callback for a given key.
+
+As this callback gets called less often then the C<filter_json_object>
+one, decoding speed will not usually suffer as much. Therefore, single-key
+objects make excellent targets to serialise Perl objects into, especially
+as single-key JSON objects are as close to the type-tagged value concept
+as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not
+support this in any way, so you need to make sure your data never looks
+like a serialised Perl hash.
+
+Typical names for the single object key are C<__class_whatever__>, or
+C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even
+things like C<__class_md5sum(classname)__>, to reduce the risk of clashing
+with real hashes.
+
+Example, decode JSON objects of the form C<< { "__widget__" => <id> } >>
+into the corresponding C<< $WIDGET{<id>} >> object:
+
+   # return whatever is in $WIDGET{5}:
+   JSON
+      ->new
+      ->filter_json_single_key_object (__widget__ => sub {
+            $WIDGET{ $_[0] }
+         })
+      ->decode ('{"__widget__": 5')
+
+   # this can be used with a TO_JSON method in some "widget" class
+   # for serialisation to json:
+   sub WidgetBase::TO_JSON {
+      my ($self) = @_;
+
+      unless ($self->{id}) {
+         $self->{id} = ..get..some..id..;
+         $WIDGET{$self->{id}} = $self;
+      }
+
+      { __widget__ => $self->{id} }
+   }
+
+
+=head2 shrink
+
+    $json = $json->shrink([$enable])
+    
+    $enabled = $json->get_shrink
+
+With JSON::XS, this flag resizes strings generated by either
+C<encode> or C<decode> to their minimum size possible. This can save
+memory when your JSON texts are either very very long or you have many
+short strings. It will also try to downgrade any strings to octet-form
+if possible: perl stores strings internally either in an encoding called
+UTF-X or in octet-form. The latter cannot store everything but uses less
+space in general (and some buggy Perl or C code might even rely on that
+internal representation being used).
+
+With JSON::PP, it is noop about resizing strings but tries
+C<utf8::downgrade> to the returned string by C<encode>. See to L<utf8>.
+
+See to L<JSON::XS/OBJECT-ORIENTED INTERFACE> and L<JSON::PP/METHODS>.
+
+=head2 max_depth
+
+    $json = $json->max_depth([$maximum_nesting_depth])
+    
+    $max_depth = $json->get_max_depth
+
+Sets the maximum nesting level (default C<512>) accepted while encoding
+or decoding. If a higher nesting level is detected in JSON text or a Perl
+data structure, then the encoder and decoder will stop and croak at that
+point.
+
+Nesting level is defined by number of hash- or arrayrefs that the encoder
+needs to traverse to reach a given point or the number of C<{> or C<[>
+characters without their matching closing parenthesis crossed to reach a
+given character in a string.
+
+If no argument is given, the highest possible setting will be used, which
+is rarely useful.
+
+Note that nesting is implemented by recursion in C. The default value has
+been chosen to be as large as typical operating systems allow without
+crashing. (JSON::XS)
+
+With JSON::PP as the backend, when a large value (100 or more) was set and
+it de/encodes a deep nested object/text, it may raise a warning
+'Deep recursion on subroutine' at the perl runtime phase.
+
+See L<JSON::XS/SECURITY CONSIDERATIONS> for more info on why this is useful.
+
+=head2 max_size
+
+    $json = $json->max_size([$maximum_string_size])
+    
+    $max_size = $json->get_max_size
+
+Set the maximum length a JSON text may have (in bytes) where decoding is
+being attempted. The default is C<0>, meaning no limit. When C<decode>
+is called on a string that is longer then this many bytes, it will not
+attempt to decode the string but throw an exception. This setting has no
+effect on C<encode> (yet).
+
+If no argument is given, the limit check will be deactivated (same as when
+C<0> is specified).
+
+See L<JSON::XS/SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
+
+=head2 encode
+
+    $json_text = $json->encode($perl_scalar)
+
+Converts the given Perl data structure (a simple scalar or a reference
+to a hash or array) to its JSON representation. Simple scalars will be
+converted into JSON string or number sequences, while references to arrays
+become JSON arrays and references to hashes become JSON objects. Undefined
+Perl values (e.g. C<undef>) become JSON C<null> values.
+References to the integers C<0> and C<1> are converted into C<true> and C<false>.
+
+=head2 decode
+
+    $perl_scalar = $json->decode($json_text)
+
+The opposite of C<encode>: expects a JSON text and tries to parse it,
+returning the resulting simple scalar or reference. Croaks on error.
+
+JSON numbers and strings become simple Perl scalars. JSON arrays become
+Perl arrayrefs and JSON objects become Perl hashrefs. C<true> becomes
+C<1> (C<JSON::true>), C<false> becomes C<0> (C<JSON::false>) and
+C<null> becomes C<undef>.
+
+=head2 decode_prefix
+
+    ($perl_scalar, $characters) = $json->decode_prefix($json_text)
+
+This works like the C<decode> method, but instead of raising an exception
+when there is trailing garbage after the first JSON object, it will
+silently stop parsing there and return the number of characters consumed
+so far.
+
+   JSON->new->decode_prefix ("[1] the tail")
+   => ([], 3)
+
+See to L<JSON::XS/OBJECT-ORIENTED INTERFACE>
+
+=head2 property
+
+    $boolean = $json->property($property_name)
+
+Returns a boolean value about above some properties.
+
+The available properties are C<ascii>, C<latin1>, C<utf8>,
+C<indent>,C<space_before>, C<space_after>, C<relaxed>, C<canonical>,
+C<allow_nonref>, C<allow_unknown>, C<allow_blessed>, C<convert_blessed>,
+C<shrink>, C<max_depth> and C<max_size>.
+
+   $boolean = $json->property('utf8');
+    => 0
+   $json->utf8;
+   $boolean = $json->property('utf8');
+    => 1
+
+Sets the property with a given boolean value.
+
+    $json = $json->property($property_name => $boolean);
+
+With no argument, it returns all the above properties as a hash reference.
+
+    $flag_hashref = $json->property();
+
+=head1 INCREMENTAL PARSING
+
+Most of this section are copied and modified from L<JSON::XS/INCREMENTAL PARSING>.
+
+In some cases, there is the need for incremental parsing of JSON texts.
+This module does allow you to parse a JSON stream incrementally.
+It does so by accumulating text until it has a full JSON object, which
+it then can decode. This process is similar to using C<decode_prefix>
+to see if a full JSON object is available, but is much more efficient
+(and can be implemented with a minimum of method calls).
+
+The backend module will only attempt to parse the JSON text once it is sure it
+has enough text to get a decisive result, using a very simple but
+truly incremental parser. This means that it sometimes won't stop as
+early as the full parser, for example, it doesn't detect parenthesis
+mismatches. The only thing it guarantees is that it starts decoding as
+soon as a syntactically valid JSON text has been seen. This means you need
+to set resource limits (e.g. C<max_size>) to ensure the parser will stop
+parsing in the presence if syntax errors.
+
+The following methods implement this incremental parser.
+
+=head2 incr_parse
+
+    $json->incr_parse( [$string] ) # void context
+    
+    $obj_or_undef = $json->incr_parse( [$string] ) # scalar context
+    
+    @obj_or_empty = $json->incr_parse( [$string] ) # list context
+
+This is the central parsing function. It can both append new text and
+extract objects from the stream accumulated so far (both of these
+functions are optional).
+
+If C<$string> is given, then this string is appended to the already
+existing JSON fragment stored in the C<$json> object.
+
+After that, if the function is called in void context, it will simply
+return without doing anything further. This can be used to add more text
+in as many chunks as you want.
+
+If the method is called in scalar context, then it will try to extract
+exactly I<one> JSON object. If that is successful, it will return this
+object, otherwise it will return C<undef>. If there is a parse error,
+this method will croak just as C<decode> would do (one can then use
+C<incr_skip> to skip the erroneous part). This is the most common way of
+using the method.
+
+And finally, in list context, it will try to extract as many objects
+from the stream as it can find and return them, or the empty list
+otherwise. For this to work, there must be no separators between the JSON
+objects or arrays, instead they must be concatenated back-to-back. If
+an error occurs, an exception will be raised as in the scalar context
+case. Note that in this case, any previously-parsed JSON texts will be
+lost.
+
+Example: Parse some JSON arrays/objects in a given string and return them.
+
+    my @objs = JSON->new->incr_parse ("[5][7][1,2]");
+
+=head2 incr_text
+
+    $lvalue_string = $json->incr_text
+
+This method returns the currently stored JSON fragment as an lvalue, that
+is, you can manipulate it. This I<only> works when a preceding call to
+C<incr_parse> in I<scalar context> successfully returned an object. Under
+all other circumstances you must not call this function (I mean it.
+although in simple tests it might actually work, it I<will> fail under
+real world conditions). As a special exception, you can also call this
+method before having parsed anything.
+
+This function is useful in two cases: a) finding the trailing text after a
+JSON object or b) parsing multiple JSON objects separated by non-JSON text
+(such as commas).
+
+    $json->incr_text =~ s/\s*,\s*//;
+
+In Perl 5.005, C<lvalue> attribute is not available.
+You must write codes like the below:
+
+    $string = $json->incr_text;
+    $string =~ s/\s*,\s*//;
+    $json->incr_text( $string );
+
+=head2 incr_skip
+
+    $json->incr_skip
+
+This will reset the state of the incremental parser and will remove the
+parsed text from the input buffer. This is useful after C<incr_parse>
+died, in which case the input buffer and incremental parser state is left
+unchanged, to skip the text parsed so far and to reset the parse state.
+
+=head2 incr_reset
+
+    $json->incr_reset
+
+This completely resets the incremental parser, that is, after this call,
+it will be as if the parser had never parsed anything.
+
+This is useful if you want to repeatedly parse JSON objects and want to
+ignore any trailing data, which means you have to reset the parser after
+each successful decode.
+
+See to L<JSON::XS/INCREMENTAL PARSING> for examples.
+
+
+=head1 JSON::PP SUPPORT METHODS
+
+The below methods are JSON::PP own methods, so when C<JSON> works
+with JSON::PP (i.e. the created object is a JSON::PP object), available.
+See to L<JSON::PP/JSON::PP OWN METHODS> in detail.
+
+If you use C<JSON> with additional C<-support_by_pp>, some methods
+are available even with JSON::XS. See to L<USE PP FEATURES EVEN THOUGH XS BACKEND>.
+
+   BEING { $ENV{PERL_JSON_BACKEND} = 'JSON::XS' }
+   
+   use JSON -support_by_pp;
+   
+   my $json = JSON->new;
+   $json->allow_nonref->escape_slash->encode("/");
+
+   # functional interfaces too.
+   print to_json(["/"], {escape_slash => 1});
+   print from_json('["foo"]', {utf8 => 1});
+
+If you do not want to all functions but C<-support_by_pp>,
+use C<-no_export>.
+
+   use JSON -support_by_pp, -no_export;
+   # functional interfaces are not exported.
+
+=head2 allow_singlequote
+
+    $json = $json->allow_singlequote([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+any JSON strings quoted by single quotations that are invalid JSON
+format.
+
+    $json->allow_singlequote->decode({"foo":'bar'});
+    $json->allow_singlequote->decode({'foo':"bar"});
+    $json->allow_singlequote->decode({'foo':'bar'});
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+=head2 allow_barekey
+
+    $json = $json->allow_barekey([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+bare keys of JSON object that are invalid JSON format.
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+    $json->allow_barekey->decode('{foo:"bar"}');
+
+=head2 allow_bignum
+
+    $json = $json->allow_bignum([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will convert
+the big integer Perl cannot handle as integer into a L<Math::BigInt>
+object and convert a floating number (any) into a L<Math::BigFloat>.
+
+On the contrary, C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers with C<allow_blessed> enable.
+
+   $json->allow_nonref->allow_blessed->allow_bignum;
+   $bigfloat = $json->decode('2.000000000000000000000000001');
+   print $json->encode($bigfloat);
+   # => 2.000000000000000000000000001
+
+See to L<MAPPING> about the conversion of JSON number.
+
+=head2 loose
+
+    $json = $json->loose([$enable])
+
+The unescaped [\x00-\x1f\x22\x2f\x5c] strings are invalid in JSON strings
+and the module doesn't allow to C<decode> to these (except for \x2f).
+If C<$enable> is true (or missing), then C<decode>  will accept these
+unescaped strings.
+
+    $json->loose->decode(qq|["abc
+                                   def"]|);
+
+See to L<JSON::PP/JSON::PP OWN METHODS>.
+
+=head2 escape_slash
+
+    $json = $json->escape_slash([$enable])
+
+According to JSON Grammar, I<slash> (U+002F) is escaped. But by default
+JSON backend modules encode strings without escaping slash.
+
+If C<$enable> is true (or missing), then C<encode> will escape slashes.
+
+=head2 indent_length
+
+    $json = $json->indent_length($length)
+
+With JSON::XS, The indent space length is 3 and cannot be changed.
+With JSON::PP, it sets the indent space length with the given $length.
+The default is 3. The acceptable range is 0 to 15.
+
+=head2 sort_by
+
+    $json = $json->sort_by($function_name)
+    $json = $json->sort_by($subroutine_ref)
+
+If $function_name or $subroutine_ref are set, its sort routine are used.
+
+   $js = $pc->sort_by(sub { $JSON::PP::a cmp $JSON::PP::b })->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   $js = $pc->sort_by('own_sort')->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   sub JSON::PP::own_sort { $JSON::PP::a cmp $JSON::PP::b }
+
+As the sorting routine runs in the JSON::PP scope, the given
+subroutine name and the special variables C<$a>, C<$b> will begin
+with 'JSON::PP::'.
+
+If $integer is set, then the effect is same as C<canonical> on.
+
+See to L<JSON::PP/JSON::PP OWN METHODS>.
+
+=head1 MAPPING
+
+This section is copied from JSON::XS and modified to C<JSON>.
+JSON::XS and JSON::PP mapping mechanisms are almost equivalent.
+
+See to L<JSON::XS/MAPPING>.
+
+=head2 JSON -> PERL
+
+=over 4
+
+=item object
+
+A JSON object becomes a reference to a hash in Perl. No ordering of object
+keys is preserved (JSON does not preserver object key ordering itself).
+
+=item array
+
+A JSON array becomes a reference to an array in Perl.
+
+=item string
+
+A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON
+are represented by the same codepoints in the Perl string, so no manual
+decoding is necessary.
+
+=item number
+
+A JSON number becomes either an integer, numeric (floating point) or
+string scalar in perl, depending on its range and any fractional parts. On
+the Perl level, there is no difference between those as Perl handles all
+the conversion details, but an integer may take slightly less memory and
+might represent more values exactly than floating point numbers.
+
+If the number consists of digits only, C<JSON> will try to represent
+it as an integer value. If that fails, it will try to represent it as
+a numeric (floating point) value if that is possible without loss of
+precision. Otherwise it will preserve the number as a string value (in
+which case you lose roundtripping ability, as the JSON number will be
+re-encoded to a JSON string).
+
+Numbers containing a fractional or exponential part will always be
+represented as numeric (floating point) values, possibly at a loss of
+precision (in which case you might lose perfect roundtripping ability, but
+the JSON number will still be re-encoded as a JSON number).
+
+Note that precision is not accuracy - binary floating point values cannot
+represent most decimal fractions exactly, and when converting from and to
+floating point, C<JSON> only guarantees precision up to but not including
+the least significant bit.
+
+If the backend is JSON::PP and C<allow_bignum> is enable, the big integers 
+and the numeric can be optionally converted into L<Math::BigInt> and
+L<Math::BigFloat> objects.
+
+=item true, false
+
+These JSON atoms become C<JSON::true> and C<JSON::false>,
+respectively. They are overloaded to act almost exactly like the numbers
+C<1> and C<0>. You can check whether a scalar is a JSON boolean by using
+the C<JSON::is_bool> function.
+
+   print JSON::true + 1;
+    => 1
+
+   ok(JSON::true eq  '1');
+   ok(JSON::true == 1);
+
+C<JSON> will install these missing overloading features to the backend modules.
+
+
+=item null
+
+A JSON null atom becomes C<undef> in Perl.
+
+C<JSON::null> returns C<undef>.
+
+=back
+
+
+=head2 PERL -> JSON
+
+The mapping from Perl to JSON is slightly more difficult, as Perl is a
+truly typeless language, so we can only guess which JSON type is meant by
+a Perl value.
+
+=over 4
+
+=item hash references
+
+Perl hash references become JSON objects. As there is no inherent ordering
+in hash keys (or JSON objects), they will usually be encoded in a
+pseudo-random order that can change between runs of the same program but
+stays generally the same within a single run of a program. C<JSON>
+optionally sort the hash keys (determined by the I<canonical> flag), so
+the same data structure will serialise to the same JSON text (given same
+settings and version of JSON::XS), but this incurs a runtime overhead
+and is only rarely useful, e.g. when you want to compare some JSON text
+against another for equality.
+
+In future, the ordered object feature will be added to JSON::PP using C<tie> mechanism.
+
+
+=item array references
+
+Perl array references become JSON arrays.
+
+=item other references
+
+Other unblessed references are generally not allowed and will cause an
+exception to be thrown, except for references to the integers C<0> and
+C<1>, which get turned into C<false> and C<true> atoms in JSON. You can
+also use C<JSON::false> and C<JSON::true> to improve readability.
+
+   to_json [\0,JSON::true]      # yields [false,true]
+
+=item JSON::true, JSON::false, JSON::null
+
+These special values become JSON true and JSON false values,
+respectively. You can also use C<\1> and C<\0> directly if you want.
+
+JSON::null returns C<undef>.
+
+=item blessed objects
+
+Blessed objects are not directly representable in JSON. See the
+C<allow_blessed> and C<convert_blessed> methods on various options on
+how to deal with this: basically, you can choose between throwing an
+exception, encoding the reference as if it weren't blessed, or provide
+your own serialiser method.
+
+With C<convert_blessed_universally> mode,  C<encode> converts blessed
+hash references or blessed array references (contains other blessed references)
+into JSON members and arrays.
+
+   use JSON -convert_blessed_universally;
+   JSON->new->allow_blessed->convert_blessed->encode( $blessed_object );
+
+See to L<convert_blessed>.
+
+=item simple scalars
+
+Simple Perl scalars (any scalar that is not a reference) are the most
+difficult objects to encode: JSON::XS and JSON::PP will encode undefined scalars as
+JSON C<null> values, scalars that have last been used in a string context
+before encoding as JSON strings, and anything else as number value:
+
+   # dump as number
+   encode_json [2]                      # yields [2]
+   encode_json [-3.0e17]                # yields [-3e+17]
+   my $value = 5; encode_json [$value]  # yields [5]
+
+   # used as string, so dump as string
+   print $value;
+   encode_json [$value]                 # yields ["5"]
+
+   # undef becomes null
+   encode_json [undef]                  # yields [null]
+
+You can force the type to be a string by stringifying it:
+
+   my $x = 3.1; # some variable containing a number
+   "$x";        # stringified
+   $x .= "";    # another, more awkward way to stringify
+   print $x;    # perl does it for you, too, quite often
+
+You can force the type to be a number by numifying it:
+
+   my $x = "3"; # some variable containing a string
+   $x += 0;     # numify it, ensuring it will be dumped as a number
+   $x *= 1;     # same thing, the choice is yours.
+
+You can not currently force the type in other, less obscure, ways.
+
+Note that numerical precision has the same meaning as under Perl (so
+binary to decimal conversion follows the same rules as in Perl, which
+can differ to other languages). Also, your perl interpreter might expose
+extensions to the floating point numbers of your platform, such as
+infinities or NaN's - these cannot be represented in JSON, and it is an
+error to pass those in.
+
+=item Big Number
+
+If the backend is JSON::PP and C<allow_bignum> is enable, 
+C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers.
+
+
+=back
+
+=head1 JSON and ECMAscript
+
+See to L<JSON::XS/JSON and ECMAscript>.
+
+=head1 JSON and YAML
+
+JSON is not a subset of YAML.
+See to L<JSON::XS/JSON and YAML>.
+
+
+=head1 BACKEND MODULE DECISION
+
+When you use C<JSON>, C<JSON> tries to C<use> JSON::XS. If this call failed, it will
+C<uses> JSON::PP. The required JSON::XS version is I<2.2> or later.
+
+The C<JSON> constructor method returns an object inherited from the backend module,
+and JSON::XS object is a blessed scalar reference while JSON::PP is a blessed hash
+reference.
+
+So, your program should not depend on the backend module, especially
+returned objects should not be modified.
+
+ my $json = JSON->new; # XS or PP?
+ $json->{stash} = 'this is xs object'; # this code may raise an error!
+
+To check the backend module, there are some methods - C<backend>, C<is_pp> and C<is_xs>.
+
+  JSON->backend; # 'JSON::XS' or 'JSON::PP'
+  
+  JSON->backend->is_pp: # 0 or 1
+  
+  JSON->backend->is_xs: # 1 or 0
+  
+  $json->is_xs; # 1 or 0
+  
+  $json->is_pp; # 0 or 1
+
+
+If you set an environment variable C<PERL_JSON_BACKEND>, the calling action will be changed.
+
+=over
+
+=item PERL_JSON_BACKEND = 0 or PERL_JSON_BACKEND = 'JSON::PP'
+
+Always use JSON::PP
+
+=item PERL_JSON_BACKEND == 1 or PERL_JSON_BACKEND = 'JSON::XS,JSON::PP'
+
+(The default) Use compiled JSON::XS if it is properly compiled & installed,
+otherwise use JSON::PP.
+
+=item PERL_JSON_BACKEND == 2 or PERL_JSON_BACKEND = 'JSON::XS'
+
+Always use compiled JSON::XS, die if it isn't properly compiled & installed.
+
+=item PERL_JSON_BACKEND = 'JSON::backportPP'
+
+Always use JSON::backportPP.
+JSON::backportPP is JSON::PP back port module.
+C<JSON> includes JSON::backportPP instead of JSON::PP.
+
+=back
+
+These ideas come from L<DBI::PurePerl> mechanism.
+
+example:
+
+ BEGIN { $ENV{PERL_JSON_BACKEND} = 'JSON::PP' }
+ use JSON; # always uses JSON::PP
+
+In future, it may be able to specify another module.
+
+=head1 USE PP FEATURES EVEN THOUGH XS BACKEND
+
+Many methods are available with either JSON::XS or JSON::PP and
+when the backend module is JSON::XS, if any JSON::PP specific (i.e. JSON::XS unsupported)
+method is called, it will C<warn> and be noop.
+
+But If you C<use> C<JSON> passing the optional string C<-support_by_pp>,
+it makes a part of those unsupported methods available.
+This feature is achieved by using JSON::PP in C<de/encode>.
+
+   BEGIN { $ENV{PERL_JSON_BACKEND} = 2 } # with JSON::XS
+   use JSON -support_by_pp;
+   my $json = JSON->new;
+   $json->allow_nonref->escape_slash->encode("/");
+
+At this time, the returned object is a C<JSON::Backend::XS::Supportable>
+object (re-blessed XS object), and  by checking JSON::XS unsupported flags
+in de/encoding, can support some unsupported methods - C<loose>, C<allow_bignum>,
+C<allow_barekey>, C<allow_singlequote>, C<escape_slash> and C<indent_length>.
+
+When any unsupported methods are not enable, C<XS de/encode> will be
+used as is. The switch is achieved by changing the symbolic tables.
+
+C<-support_by_pp> is effective only when the backend module is JSON::XS
+and it makes the de/encoding speed down a bit.
+
+See to L<JSON::PP SUPPORT METHODS>.
+
+=head1 INCOMPATIBLE CHANGES TO OLD VERSION
+
+There are big incompatibility between new version (2.00) and old (1.xx).
+If you use old C<JSON> 1.xx in your code, please check it.
+
+See to L<Transition ways from 1.xx to 2.xx.>
+
+=over
+
+=item jsonToObj and objToJson are obsoleted.
+
+Non Perl-style name C<jsonToObj> and C<objToJson> are obsoleted
+(but not yet deleted from the source).
+If you use these functions in your code, please replace them
+with C<from_json> and C<to_json>.
+
+
+=item Global variables are no longer available.
+
+C<JSON> class variables - C<$JSON::AUTOCONVERT>, C<$JSON::BareKey>, etc...
+- are not available any longer.
+Instead, various features can be used through object methods.
+
+
+=item Package JSON::Converter and JSON::Parser are deleted.
+
+Now C<JSON> bundles with JSON::PP which can handle JSON more properly than them.
+
+=item Package JSON::NotString is deleted.
+
+There was C<JSON::NotString> class which represents JSON value C<true>, C<false>, C<null>
+and numbers. It was deleted and replaced by C<JSON::Boolean>.
+
+C<JSON::Boolean> represents C<true> and C<false>.
+
+C<JSON::Boolean> does not represent C<null>.
+
+C<JSON::null> returns C<undef>.
+
+C<JSON> makes L<JSON::XS::Boolean> and L<JSON::PP::Boolean> is-a relation
+to L<JSON::Boolean>.
+
+=item function JSON::Number is obsoleted.
+
+C<JSON::Number> is now needless because JSON::XS and JSON::PP have
+round-trip integrity.
+
+=item JSONRPC modules are deleted.
+
+Perl implementation of JSON-RPC protocol - C<JSONRPC >, C<JSONRPC::Transport::HTTP>
+and C<Apache::JSONRPC > are deleted in this distribution.
+Instead of them, there is L<JSON::RPC> which supports JSON-RPC protocol version 1.1.
+
+=back
+
+=head2 Transition ways from 1.xx to 2.xx.
+
+You should set C<suport_by_pp> mode firstly, because
+it is always successful for the below codes even with JSON::XS.
+
+    use JSON -support_by_pp;
+
+=over
+
+=item Exported jsonToObj (simple)
+
+  from_json($json_text);
+
+=item Exported objToJson (simple)
+
+  to_json($perl_scalar);
+
+=item Exported jsonToObj (advanced)
+
+  $flags = {allow_barekey => 1, allow_singlequote => 1};
+  from_json($json_text, $flags);
+
+equivalent to:
+
+  $JSON::BareKey = 1;
+  $JSON::QuotApos = 1;
+  jsonToObj($json_text);
+
+=item Exported objToJson (advanced)
+
+  $flags = {allow_blessed => 1, allow_barekey => 1};
+  to_json($perl_scalar, $flags);
+
+equivalent to:
+
+  $JSON::BareKey = 1;
+  objToJson($perl_scalar);
+
+=item jsonToObj as object method
+
+  $json->decode($json_text);
+
+=item objToJson as object method
+
+  $json->encode($perl_scalar);
+
+=item new method with parameters
+
+The C<new> method in 2.x takes any parameters no longer.
+You can set parameters instead;
+
+   $json = JSON->new->pretty;
+
+=item $JSON::Pretty, $JSON::Indent, $JSON::Delimiter
+
+If C<indent> is enable, that means C<$JSON::Pretty> flag set. And
+C<$JSON::Delimiter> was substituted by C<space_before> and C<space_after>.
+In conclusion:
+
+   $json->indent->space_before->space_after;
+
+Equivalent to:
+
+  $json->pretty;
+
+To change indent length, use C<indent_length>.
+
+(Only with JSON::PP, if C<-support_by_pp> is not used.)
+
+  $json->pretty->indent_length(2)->encode($perl_scalar);
+
+=item $JSON::BareKey
+
+(Only with JSON::PP, if C<-support_by_pp> is not used.)
+
+  $json->allow_barekey->decode($json_text)
+
+=item $JSON::ConvBlessed
+
+use C<-convert_blessed_universally>. See to L<convert_blessed>.
+
+=item $JSON::QuotApos
+
+(Only with JSON::PP, if C<-support_by_pp> is not used.)
+
+  $json->allow_singlequote->decode($json_text)
+
+=item $JSON::SingleQuote
+
+Disable. C<JSON> does not make such a invalid JSON string any longer.
+
+=item $JSON::KeySort
+
+  $json->canonical->encode($perl_scalar)
+
+This is the ascii sort.
+
+If you want to use with your own sort routine, check the C<sort_by> method.
+
+(Only with JSON::PP, even if C<-support_by_pp> is used currently.)
+
+  $json->sort_by($sort_routine_ref)->encode($perl_scalar)
+ 
+  $json->sort_by(sub { $JSON::PP::a <=> $JSON::PP::b })->encode($perl_scalar)
+
+Can't access C<$a> and C<$b> but C<$JSON::PP::a> and C<$JSON::PP::b>.
+
+=item $JSON::SkipInvalid
+
+  $json->allow_unknown
+
+=item $JSON::AUTOCONVERT
+
+Needless. C<JSON> backend modules have the round-trip integrity.
+
+=item $JSON::UTF8
+
+Needless because C<JSON> (JSON::XS/JSON::PP) sets
+the UTF8 flag on properly.
+
+    # With UTF8-flagged strings
+
+    $json->allow_nonref;
+    $str = chr(1000); # UTF8-flagged
+
+    $json_text  = $json->utf8(0)->encode($str);
+    utf8::is_utf8($json_text);
+    # true
+    $json_text  = $json->utf8(1)->encode($str);
+    utf8::is_utf8($json_text);
+    # false
+
+    $str = '"' . chr(1000) . '"'; # UTF8-flagged
+
+    $perl_scalar  = $json->utf8(0)->decode($str);
+    utf8::is_utf8($perl_scalar);
+    # true
+    $perl_scalar  = $json->utf8(1)->decode($str);
+    # died because of 'Wide character in subroutine'
+
+See to L<JSON::XS/A FEW NOTES ON UNICODE AND PERL>.
+
+=item $JSON::UnMapping
+
+Disable. See to L<MAPPING>.
+
+=item $JSON::SelfConvert
+
+This option was deleted.
+Instead of it, if a given blessed object has the C<TO_JSON> method,
+C<TO_JSON> will be executed with C<convert_blessed>.
+
+  $json->convert_blessed->encode($blessed_hashref_or_arrayref)
+  # if need, call allow_blessed
+
+Note that it was C<toJson> in old version, but now not C<toJson> but C<TO_JSON>.
+
+=back
+
+=head1 TODO
+
+=over
+
+=item example programs
+
+=back
+
+=head1 THREADS
+
+No test with JSON::PP. If with JSON::XS, See to L<JSON::XS/THREADS>.
+
+
+=head1 BUGS
+
+Please report bugs relevant to C<JSON> to E<lt>makamaka[at]cpan.orgE<gt>.
+
+
+=head1 SEE ALSO
+
+Most of the document is copied and modified from JSON::XS doc.
+
+L<JSON::XS>, L<JSON::PP>
+
+C<RFC4627>(L<http://www.ietf.org/rfc/rfc4627.txt>)
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+JSON::XS was written by  Marc Lehmann <schmorp[at]schmorp.de>
+
+The release of this new version owes to the courtesy of Marc Lehmann.
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2005-2013 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut
+

uroman/lib/JSON/backportPP.pm

@@ -0,0 +1,2806 @@
+package # This is JSON::backportPP
+    JSON::PP;
+
+# JSON-2.0
+
+use 5.005;
+use strict;
+use base qw(Exporter);
+use overload ();
+
+use Carp ();
+use B ();
+#use Devel::Peek;
+
+use vars qw($VERSION);
+$VERSION = '2.27204';
+
+@JSON::PP::EXPORT = qw(encode_json decode_json from_json to_json);
+
+# instead of hash-access, i tried index-access for speed.
+# but this method is not faster than what i expected. so it will be changed.
+
+use constant P_ASCII                => 0;
+use constant P_LATIN1               => 1;
+use constant P_UTF8                 => 2;
+use constant P_INDENT               => 3;
+use constant P_CANONICAL            => 4;
+use constant P_SPACE_BEFORE         => 5;
+use constant P_SPACE_AFTER          => 6;
+use constant P_ALLOW_NONREF         => 7;
+use constant P_SHRINK               => 8;
+use constant P_ALLOW_BLESSED        => 9;
+use constant P_CONVERT_BLESSED      => 10;
+use constant P_RELAXED              => 11;
+
+use constant P_LOOSE                => 12;
+use constant P_ALLOW_BIGNUM         => 13;
+use constant P_ALLOW_BAREKEY        => 14;
+use constant P_ALLOW_SINGLEQUOTE    => 15;
+use constant P_ESCAPE_SLASH         => 16;
+use constant P_AS_NONBLESSED        => 17;
+
+use constant P_ALLOW_UNKNOWN        => 18;
+
+use constant OLD_PERL => $] < 5.008 ? 1 : 0;
+
+BEGIN {
+    my @xs_compati_bit_properties = qw(
+            latin1 ascii utf8 indent canonical space_before space_after allow_nonref shrink
+            allow_blessed convert_blessed relaxed allow_unknown
+    );
+    my @pp_bit_properties = qw(
+            allow_singlequote allow_bignum loose
+            allow_barekey escape_slash as_nonblessed
+    );
+
+    # Perl version check, Unicode handling is enable?
+    # Helper module sets @JSON::PP::_properties.
+    if ($] < 5.008 ) {
+        my $helper = $] >= 5.006 ? 'JSON::backportPP::Compat5006' : 'JSON::backportPP::Compat5005';
+        eval qq| require $helper |;
+        if ($@) { Carp::croak $@; }
+    }
+
+    for my $name (@xs_compati_bit_properties, @pp_bit_properties) {
+        my $flag_name = 'P_' . uc($name);
+
+        eval qq/
+            sub $name {
+                my \$enable = defined \$_[1] ? \$_[1] : 1;
+
+                if (\$enable) {
+                    \$_[0]->{PROPS}->[$flag_name] = 1;
+                }
+                else {
+                    \$_[0]->{PROPS}->[$flag_name] = 0;
+                }
+
+                \$_[0];
+            }
+
+            sub get_$name {
+                \$_[0]->{PROPS}->[$flag_name] ? 1 : '';
+            }
+        /;
+    }
+
+}
+
+
+
+# Functions
+
+my %encode_allow_method
+     = map {($_ => 1)} qw/utf8 pretty allow_nonref latin1 self_encode escape_slash
+                          allow_blessed convert_blessed indent indent_length allow_bignum
+                          as_nonblessed
+                        /;
+my %decode_allow_method
+     = map {($_ => 1)} qw/utf8 allow_nonref loose allow_singlequote allow_bignum
+                          allow_barekey max_size relaxed/;
+
+
+my $JSON; # cache
+
+sub encode_json ($) { # encode
+    ($JSON ||= __PACKAGE__->new->utf8)->encode(@_);
+}
+
+
+sub decode_json { # decode
+    ($JSON ||= __PACKAGE__->new->utf8)->decode(@_);
+}
+
+# Obsoleted
+
+sub to_json($) {
+   Carp::croak ("JSON::PP::to_json has been renamed to encode_json.");
+}
+
+
+sub from_json($) {
+   Carp::croak ("JSON::PP::from_json has been renamed to decode_json.");
+}
+
+
+# Methods
+
+sub new {
+    my $class = shift;
+    my $self  = {
+        max_depth   => 512,
+        max_size    => 0,
+        indent      => 0,
+        FLAGS       => 0,
+        fallback      => sub { encode_error('Invalid value. JSON can only reference.') },
+        indent_length => 3,
+    };
+
+    bless $self, $class;
+}
+
+
+sub encode {
+    return $_[0]->PP_encode_json($_[1]);
+}
+
+
+sub decode {
+    return $_[0]->PP_decode_json($_[1], 0x00000000);
+}
+
+
+sub decode_prefix {
+    return $_[0]->PP_decode_json($_[1], 0x00000001);
+}
+
+
+# accessor
+
+
+# pretty printing
+
+sub pretty {
+    my ($self, $v) = @_;
+    my $enable = defined $v ? $v : 1;
+
+    if ($enable) { # indent_length(3) for JSON::XS compatibility
+        $self->indent(1)->indent_length(3)->space_before(1)->space_after(1);
+    }
+    else {
+        $self->indent(0)->space_before(0)->space_after(0);
+    }
+
+    $self;
+}
+
+# etc
+
+sub max_depth {
+    my $max  = defined $_[1] ? $_[1] : 0x80000000;
+    $_[0]->{max_depth} = $max;
+    $_[0];
+}
+
+
+sub get_max_depth { $_[0]->{max_depth}; }
+
+
+sub max_size {
+    my $max  = defined $_[1] ? $_[1] : 0;
+    $_[0]->{max_size} = $max;
+    $_[0];
+}
+
+
+sub get_max_size { $_[0]->{max_size}; }
+
+
+sub filter_json_object {
+    $_[0]->{cb_object} = defined $_[1] ? $_[1] : 0;
+    $_[0]->{F_HOOK} = ($_[0]->{cb_object} or $_[0]->{cb_sk_object}) ? 1 : 0;
+    $_[0];
+}
+
+sub filter_json_single_key_object {
+    if (@_ > 1) {
+        $_[0]->{cb_sk_object}->{$_[1]} = $_[2];
+    }
+    $_[0]->{F_HOOK} = ($_[0]->{cb_object} or $_[0]->{cb_sk_object}) ? 1 : 0;
+    $_[0];
+}
+
+sub indent_length {
+    if (!defined $_[1] or $_[1] > 15 or $_[1] < 0) {
+        Carp::carp "The acceptable range of indent_length() is 0 to 15.";
+    }
+    else {
+        $_[0]->{indent_length} = $_[1];
+    }
+    $_[0];
+}
+
+sub get_indent_length {
+    $_[0]->{indent_length};
+}
+
+sub sort_by {
+    $_[0]->{sort_by} = defined $_[1] ? $_[1] : 1;
+    $_[0];
+}
+
+sub allow_bigint {
+    Carp::carp("allow_bigint() is obsoleted. use allow_bignum() insted.");
+}
+
+###############################
+
+###
+### Perl => JSON
+###
+
+
+{ # Convert
+
+    my $max_depth;
+    my $indent;
+    my $ascii;
+    my $latin1;
+    my $utf8;
+    my $space_before;
+    my $space_after;
+    my $canonical;
+    my $allow_blessed;
+    my $convert_blessed;
+
+    my $indent_length;
+    my $escape_slash;
+    my $bignum;
+    my $as_nonblessed;
+
+    my $depth;
+    my $indent_count;
+    my $keysort;
+
+
+    sub PP_encode_json {
+        my $self = shift;
+        my $obj  = shift;
+
+        $indent_count = 0;
+        $depth        = 0;
+
+        my $idx = $self->{PROPS};
+
+        ($ascii, $latin1, $utf8, $indent, $canonical, $space_before, $space_after, $allow_blessed,
+            $convert_blessed, $escape_slash, $bignum, $as_nonblessed)
+         = @{$idx}[P_ASCII .. P_SPACE_AFTER, P_ALLOW_BLESSED, P_CONVERT_BLESSED,
+                    P_ESCAPE_SLASH, P_ALLOW_BIGNUM, P_AS_NONBLESSED];
+
+        ($max_depth, $indent_length) = @{$self}{qw/max_depth indent_length/};
+
+        $keysort = $canonical ? sub { $a cmp $b } : undef;
+
+        if ($self->{sort_by}) {
+            $keysort = ref($self->{sort_by}) eq 'CODE' ? $self->{sort_by}
+                     : $self->{sort_by} =~ /\D+/       ? $self->{sort_by}
+                     : sub { $a cmp $b };
+        }
+
+        encode_error("hash- or arrayref expected (not a simple scalar, use allow_nonref to allow this)")
+             if(!ref $obj and !$idx->[ P_ALLOW_NONREF ]);
+
+        my $str  = $self->object_to_json($obj);
+
+        $str .= "\n" if ( $indent ); # JSON::XS 2.26 compatible
+
+        unless ($ascii or $latin1 or $utf8) {
+            utf8::upgrade($str);
+        }
+
+        if ($idx->[ P_SHRINK ]) {
+            utf8::downgrade($str, 1);
+        }
+
+        return $str;
+    }
+
+
+    sub object_to_json {
+        my ($self, $obj) = @_;
+        my $type = ref($obj);
+
+        if($type eq 'HASH'){
+            return $self->hash_to_json($obj);
+        }
+        elsif($type eq 'ARRAY'){
+            return $self->array_to_json($obj);
+        }
+        elsif ($type) { # blessed object?
+            if (blessed($obj)) {
+
+                return $self->value_to_json($obj) if ( $obj->isa('JSON::PP::Boolean') );
+
+                if ( $convert_blessed and $obj->can('TO_JSON') ) {
+                    my $result = $obj->TO_JSON();
+                    if ( defined $result and ref( $result ) ) {
+                        if ( refaddr( $obj ) eq refaddr( $result ) ) {
+                            encode_error( sprintf(
+                                "%s::TO_JSON method returned same object as was passed instead of a new one",
+                                ref $obj
+                            ) );
+                        }
+                    }
+
+                    return $self->object_to_json( $result );
+                }
+
+                return "$obj" if ( $bignum and _is_bignum($obj) );
+                return $self->blessed_to_json($obj) if ($allow_blessed and $as_nonblessed); # will be removed.
+
+                encode_error( sprintf("encountered object '%s', but neither allow_blessed "
+                    . "nor convert_blessed settings are enabled", $obj)
+                ) unless ($allow_blessed);
+
+                return 'null';
+            }
+            else {
+                return $self->value_to_json($obj);
+            }
+        }
+        else{
+            return $self->value_to_json($obj);
+        }
+    }
+
+
+    sub hash_to_json {
+        my ($self, $obj) = @_;
+        my @res;
+
+        encode_error("json text or perl structure exceeds maximum nesting level (max_depth set too low?)")
+                                         if (++$depth > $max_depth);
+
+        my ($pre, $post) = $indent ? $self->_up_indent() : ('', '');
+        my $del = ($space_before ? ' ' : '') . ':' . ($space_after ? ' ' : '');
+
+        for my $k ( _sort( $obj ) ) {
+            if ( OLD_PERL ) { utf8::decode($k) } # key for Perl 5.6 / be optimized
+            push @res, string_to_json( $self, $k )
+                          .  $del
+                          . ( $self->object_to_json( $obj->{$k} ) || $self->value_to_json( $obj->{$k} ) );
+        }
+
+        --$depth;
+        $self->_down_indent() if ($indent);
+
+        return   '{' . ( @res ? $pre : '' ) . ( @res ? join( ",$pre", @res ) . $post : '' )  . '}';
+    }
+
+
+    sub array_to_json {
+        my ($self, $obj) = @_;
+        my @res;
+
+        encode_error("json text or perl structure exceeds maximum nesting level (max_depth set too low?)")
+                                         if (++$depth > $max_depth);
+
+        my ($pre, $post) = $indent ? $self->_up_indent() : ('', '');
+
+        for my $v (@$obj){
+            push @res, $self->object_to_json($v) || $self->value_to_json($v);
+        }
+
+        --$depth;
+        $self->_down_indent() if ($indent);
+
+        return '[' . ( @res ? $pre : '' ) . ( @res ? join( ",$pre", @res ) . $post : '' ) . ']';
+    }
+
+
+    sub value_to_json {
+        my ($self, $value) = @_;
+
+        return 'null' if(!defined $value);
+
+        my $b_obj = B::svref_2object(\$value);  # for round trip problem
+        my $flags = $b_obj->FLAGS;
+
+        return $value # as is 
+            if $flags & ( B::SVp_IOK | B::SVp_NOK ) and !( $flags & B::SVp_POK ); # SvTYPE is IV or NV?
+
+        my $type = ref($value);
+
+        if(!$type){
+            return string_to_json($self, $value);
+        }
+        elsif( blessed($value) and  $value->isa('JSON::PP::Boolean') ){
+            return $$value == 1 ? 'true' : 'false';
+        }
+        elsif ($type) {
+            if ((overload::StrVal($value) =~ /=(\w+)/)[0]) {
+                return $self->value_to_json("$value");
+            }
+
+            if ($type eq 'SCALAR' and defined $$value) {
+                return   $$value eq '1' ? 'true'
+                       : $$value eq '0' ? 'false'
+                       : $self->{PROPS}->[ P_ALLOW_UNKNOWN ] ? 'null'
+                       : encode_error("cannot encode reference to scalar");
+            }
+
+             if ( $self->{PROPS}->[ P_ALLOW_UNKNOWN ] ) {
+                 return 'null';
+             }
+             else {
+                 if ( $type eq 'SCALAR' or $type eq 'REF' ) {
+                    encode_error("cannot encode reference to scalar");
+                 }
+                 else {
+                    encode_error("encountered $value, but JSON can only represent references to arrays or hashes");
+                 }
+             }
+
+        }
+        else {
+            return $self->{fallback}->($value)
+                 if ($self->{fallback} and ref($self->{fallback}) eq 'CODE');
+            return 'null';
+        }
+
+    }
+
+
+    my %esc = (
+        "\n" => '\n',
+        "\r" => '\r',
+        "\t" => '\t',
+        "\f" => '\f',
+        "\b" => '\b',
+        "\"" => '\"',
+        "\\" => '\\\\',
+        "\'" => '\\\'',
+    );
+
+
+    sub string_to_json {
+        my ($self, $arg) = @_;
+
+        $arg =~ s/([\x22\x5c\n\r\t\f\b])/$esc{$1}/g;
+        $arg =~ s/\//\\\//g if ($escape_slash);
+        $arg =~ s/([\x00-\x08\x0b\x0e-\x1f])/'\\u00' . unpack('H2', $1)/eg;
+
+        if ($ascii) {
+            $arg = JSON_PP_encode_ascii($arg);
+        }
+
+        if ($latin1) {
+            $arg = JSON_PP_encode_latin1($arg);
+        }
+
+        if ($utf8) {
+            utf8::encode($arg);
+        }
+
+        return '"' . $arg . '"';
+    }
+
+
+    sub blessed_to_json {
+        my $reftype = reftype($_[1]) || '';
+        if ($reftype eq 'HASH') {
+            return $_[0]->hash_to_json($_[1]);
+        }
+        elsif ($reftype eq 'ARRAY') {
+            return $_[0]->array_to_json($_[1]);
+        }
+        else {
+            return 'null';
+        }
+    }
+
+
+    sub encode_error {
+        my $error  = shift;
+        Carp::croak "$error";
+    }
+
+
+    sub _sort {
+        defined $keysort ? (sort $keysort (keys %{$_[0]})) : keys %{$_[0]};
+    }
+
+
+    sub _up_indent {
+        my $self  = shift;
+        my $space = ' ' x $indent_length;
+
+        my ($pre,$post) = ('','');
+
+        $post = "\n" . $space x $indent_count;
+
+        $indent_count++;
+
+        $pre = "\n" . $space x $indent_count;
+
+        return ($pre,$post);
+    }
+
+
+    sub _down_indent { $indent_count--; }
+
+
+    sub PP_encode_box {
+        {
+            depth        => $depth,
+            indent_count => $indent_count,
+        };
+    }
+
+} # Convert
+
+
+sub _encode_ascii {
+    join('',
+        map {
+            $_ <= 127 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', _encode_surrogates($_));
+        } unpack('U*', $_[0])
+    );
+}
+
+
+sub _encode_latin1 {
+    join('',
+        map {
+            $_ <= 255 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', _encode_surrogates($_));
+        } unpack('U*', $_[0])
+    );
+}
+
+
+sub _encode_surrogates { # from perlunicode
+    my $uni = $_[0] - 0x10000;
+    return ($uni / 0x400 + 0xD800, $uni % 0x400 + 0xDC00);
+}
+
+
+sub _is_bignum {
+    $_[0]->isa('Math::BigInt') or $_[0]->isa('Math::BigFloat');
+}
+
+
+
+#
+# JSON => Perl
+#
+
+my $max_intsize;
+
+BEGIN {
+    my $checkint = 1111;
+    for my $d (5..64) {
+        $checkint .= 1;
+        my $int   = eval qq| $checkint |;
+        if ($int =~ /[eE]/) {
+            $max_intsize = $d - 1;
+            last;
+        }
+    }
+}
+
+{ # PARSE 
+
+    my %escapes = ( #  by Jeremy Muhlich <jmuhlich [at] bitflood.org>
+        b    => "\x8",
+        t    => "\x9",
+        n    => "\xA",
+        f    => "\xC",
+        r    => "\xD",
+        '\\' => '\\',
+        '"'  => '"',
+        '/'  => '/',
+    );
+
+    my $text; # json data
+    my $at;   # offset
+    my $ch;   # 1chracter
+    my $len;  # text length (changed according to UTF8 or NON UTF8)
+    # INTERNAL
+    my $depth;          # nest counter
+    my $encoding;       # json text encoding
+    my $is_valid_utf8;  # temp variable
+    my $utf8_len;       # utf8 byte length
+    # FLAGS
+    my $utf8;           # must be utf8
+    my $max_depth;      # max nest number of objects and arrays
+    my $max_size;
+    my $relaxed;
+    my $cb_object;
+    my $cb_sk_object;
+
+    my $F_HOOK;
+
+    my $allow_bigint;   # using Math::BigInt
+    my $singlequote;    # loosely quoting
+    my $loose;          # 
+    my $allow_barekey;  # bareKey
+
+    # $opt flag
+    # 0x00000001 .... decode_prefix
+    # 0x10000000 .... incr_parse
+
+    sub PP_decode_json {
+        my ($self, $opt); # $opt is an effective flag during this decode_json.
+
+        ($self, $text, $opt) = @_;
+
+        ($at, $ch, $depth) = (0, '', 0);
+
+        if ( !defined $text or ref $text ) {
+            decode_error("malformed JSON string, neither array, object, number, string or atom");
+        }
+
+        my $idx = $self->{PROPS};
+
+        ($utf8, $relaxed, $loose, $allow_bigint, $allow_barekey, $singlequote)
+            = @{$idx}[P_UTF8, P_RELAXED, P_LOOSE .. P_ALLOW_SINGLEQUOTE];
+
+        if ( $utf8 ) {
+            utf8::downgrade( $text, 1 ) or Carp::croak("Wide character in subroutine entry");
+        }
+        else {
+            utf8::upgrade( $text );
+        }
+
+        $len = length $text;
+
+        ($max_depth, $max_size, $cb_object, $cb_sk_object, $F_HOOK)
+             = @{$self}{qw/max_depth  max_size cb_object cb_sk_object F_HOOK/};
+
+        if ($max_size > 1) {
+            use bytes;
+            my $bytes = length $text;
+            decode_error(
+                sprintf("attempted decode of JSON text of %s bytes size, but max_size is set to %s"
+                    , $bytes, $max_size), 1
+            ) if ($bytes > $max_size);
+        }
+
+        # Currently no effect
+        # should use regexp
+        my @octets = unpack('C4', $text);
+        $encoding =   ( $octets[0] and  $octets[1]) ? 'UTF-8'
+                    : (!$octets[0] and  $octets[1]) ? 'UTF-16BE'
+                    : (!$octets[0] and !$octets[1]) ? 'UTF-32BE'
+                    : ( $octets[2]                ) ? 'UTF-16LE'
+                    : (!$octets[2]                ) ? 'UTF-32LE'
+                    : 'unknown';
+
+        white(); # remove head white space
+
+        my $valid_start = defined $ch; # Is there a first character for JSON structure?
+
+        my $result = value();
+
+        return undef if ( !$result && ( $opt & 0x10000000 ) ); # for incr_parse
+
+        decode_error("malformed JSON string, neither array, object, number, string or atom") unless $valid_start;
+
+        if ( !$idx->[ P_ALLOW_NONREF ] and !ref $result ) {
+                decode_error(
+                'JSON text must be an object or array (but found number, string, true, false or null,'
+                       . ' use allow_nonref to allow this)', 1);
+        }
+
+        Carp::croak('something wrong.') if $len < $at; # we won't arrive here.
+
+        my $consumed = defined $ch ? $at - 1 : $at; # consumed JSON text length
+
+        white(); # remove tail white space
+
+        if ( $ch ) {
+            return ( $result, $consumed ) if ($opt & 0x00000001); # all right if decode_prefix
+            decode_error("garbage after JSON object");
+        }
+
+        ( $opt & 0x00000001 ) ? ( $result, $consumed ) : $result;
+    }
+
+
+    sub next_chr {
+        return $ch = undef if($at >= $len);
+        $ch = substr($text, $at++, 1);
+    }
+
+
+    sub value {
+        white();
+        return          if(!defined $ch);
+        return object() if($ch eq '{');
+        return array()  if($ch eq '[');
+        return string() if($ch eq '"' or ($singlequote and $ch eq "'"));
+        return number() if($ch =~ /[0-9]/ or $ch eq '-');
+        return word();
+    }
+
+    sub string {
+        my ($i, $s, $t, $u);
+        my $utf16;
+        my $is_utf8;
+
+        ($is_valid_utf8, $utf8_len) = ('', 0);
+
+        $s = ''; # basically UTF8 flag on
+
+        if($ch eq '"' or ($singlequote and $ch eq "'")){
+            my $boundChar = $ch;
+
+            OUTER: while( defined(next_chr()) ){
+
+                if($ch eq $boundChar){
+                    next_chr();
+
+                    if ($utf16) {
+                        decode_error("missing low surrogate character in surrogate pair");
+                    }
+
+                    utf8::decode($s) if($is_utf8);
+
+                    return $s;
+                }
+                elsif($ch eq '\\'){
+                    next_chr();
+                    if(exists $escapes{$ch}){
+                        $s .= $escapes{$ch};
+                    }
+                    elsif($ch eq 'u'){ # UNICODE handling
+                        my $u = '';
+
+                        for(1..4){
+                            $ch = next_chr();
+                            last OUTER if($ch !~ /[0-9a-fA-F]/);
+                            $u .= $ch;
+                        }
+
+                        # U+D800 - U+DBFF
+                        if ($u =~ /^[dD][89abAB][0-9a-fA-F]{2}/) { # UTF-16 high surrogate?
+                            $utf16 = $u;
+                        }
+                        # U+DC00 - U+DFFF
+                        elsif ($u =~ /^[dD][c-fC-F][0-9a-fA-F]{2}/) { # UTF-16 low surrogate?
+                            unless (defined $utf16) {
+                                decode_error("missing high surrogate character in surrogate pair");
+                            }
+                            $is_utf8 = 1;
+                            $s .= JSON_PP_decode_surrogates($utf16, $u) || next;
+                            $utf16 = undef;
+                        }
+                        else {
+                            if (defined $utf16) {
+                                decode_error("surrogate pair expected");
+                            }
+
+                            if ( ( my $hex = hex( $u ) ) > 127 ) {
+                                $is_utf8 = 1;
+                                $s .= JSON_PP_decode_unicode($u) || next;
+                            }
+                            else {
+                                $s .= chr $hex;
+                            }
+                        }
+
+                    }
+                    else{
+                        unless ($loose) {
+                            $at -= 2;
+                            decode_error('illegal backslash escape sequence in string');
+                        }
+                        $s .= $ch;
+                    }
+                }
+                else{
+
+                    if ( ord $ch  > 127 ) {
+                        if ( $utf8 ) {
+                            unless( $ch = is_valid_utf8($ch) ) {
+                                $at -= 1;
+                                decode_error("malformed UTF-8 character in JSON string");
+                            }
+                            else {
+                                $at += $utf8_len - 1;
+                            }
+                        }
+                        else {
+                            utf8::encode( $ch );
+                        }
+
+                        $is_utf8 = 1;
+                    }
+
+                    if (!$loose) {
+                        if ($ch =~ /[\x00-\x1f\x22\x5c]/)  { # '/' ok
+                            $at--;
+                            decode_error('invalid character encountered while parsing JSON string');
+                        }
+                    }
+
+                    $s .= $ch;
+                }
+            }
+        }
+
+        decode_error("unexpected end of string while parsing JSON string");
+    }
+
+
+    sub white {
+        while( defined $ch  ){
+            if($ch le ' '){
+                next_chr();
+            }
+            elsif($ch eq '/'){
+                next_chr();
+                if(defined $ch and $ch eq '/'){
+                    1 while(defined(next_chr()) and $ch ne "\n" and $ch ne "\r");
+                }
+                elsif(defined $ch and $ch eq '*'){
+                    next_chr();
+                    while(1){
+                        if(defined $ch){
+                            if($ch eq '*'){
+                                if(defined(next_chr()) and $ch eq '/'){
+                                    next_chr();
+                                    last;
+                                }
+                            }
+                            else{
+                                next_chr();
+                            }
+                        }
+                        else{
+                            decode_error("Unterminated comment");
+                        }
+                    }
+                    next;
+                }
+                else{
+                    $at--;
+                    decode_error("malformed JSON string, neither array, object, number, string or atom");
+                }
+            }
+            else{
+                if ($relaxed and $ch eq '#') { # correctly?
+                    pos($text) = $at;
+                    $text =~ /\G([^\n]*(?:\r\n|\r|\n|$))/g;
+                    $at = pos($text);
+                    next_chr;
+                    next;
+                }
+
+                last;
+            }
+        }
+    }
+
+
+    sub array {
+        my $a  = $_[0] || []; # you can use this code to use another array ref object.
+
+        decode_error('json text or perl structure exceeds maximum nesting level (max_depth set too low?)')
+                                                    if (++$depth > $max_depth);
+
+        next_chr();
+        white();
+
+        if(defined $ch and $ch eq ']'){
+            --$depth;
+            next_chr();
+            return $a;
+        }
+        else {
+            while(defined($ch)){
+                push @$a, value();
+
+                white();
+
+                if (!defined $ch) {
+                    last;
+                }
+
+                if($ch eq ']'){
+                    --$depth;
+                    next_chr();
+                    return $a;
+                }
+
+                if($ch ne ','){
+                    last;
+                }
+
+                next_chr();
+                white();
+
+                if ($relaxed and $ch eq ']') {
+                    --$depth;
+                    next_chr();
+                    return $a;
+                }
+
+            }
+        }
+
+        decode_error(", or ] expected while parsing array");
+    }
+
+
+    sub object {
+        my $o = $_[0] || {}; # you can use this code to use another hash ref object.
+        my $k;
+
+        decode_error('json text or perl structure exceeds maximum nesting level (max_depth set too low?)')
+                                                if (++$depth > $max_depth);
+        next_chr();
+        white();
+
+        if(defined $ch and $ch eq '}'){
+            --$depth;
+            next_chr();
+            if ($F_HOOK) {
+                return _json_object_hook($o);
+            }
+            return $o;
+        }
+        else {
+            while (defined $ch) {
+                $k = ($allow_barekey and $ch ne '"' and $ch ne "'") ? bareKey() : string();
+                white();
+
+                if(!defined $ch or $ch ne ':'){
+                    $at--;
+                    decode_error("':' expected");
+                }
+
+                next_chr();
+                $o->{$k} = value();
+                white();
+
+                last if (!defined $ch);
+
+                if($ch eq '}'){
+                    --$depth;
+                    next_chr();
+                    if ($F_HOOK) {
+                        return _json_object_hook($o);
+                    }
+                    return $o;
+                }
+
+                if($ch ne ','){
+                    last;
+                }
+
+                next_chr();
+                white();
+
+                if ($relaxed and $ch eq '}') {
+                    --$depth;
+                    next_chr();
+                    if ($F_HOOK) {
+                        return _json_object_hook($o);
+                    }
+                    return $o;
+                }
+
+            }
+
+        }
+
+        $at--;
+        decode_error(", or } expected while parsing object/hash");
+    }
+
+
+    sub bareKey { # doesn't strictly follow Standard ECMA-262 3rd Edition
+        my $key;
+        while($ch =~ /[^\x00-\x23\x25-\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\x7F]/){
+            $key .= $ch;
+            next_chr();
+        }
+        return $key;
+    }
+
+
+    sub word {
+        my $word =  substr($text,$at-1,4);
+
+        if($word eq 'true'){
+            $at += 3;
+            next_chr;
+            return $JSON::PP::true;
+        }
+        elsif($word eq 'null'){
+            $at += 3;
+            next_chr;
+            return undef;
+        }
+        elsif($word eq 'fals'){
+            $at += 3;
+            if(substr($text,$at,1) eq 'e'){
+                $at++;
+                next_chr;
+                return $JSON::PP::false;
+            }
+        }
+
+        $at--; # for decode_error report
+
+        decode_error("'null' expected")  if ($word =~ /^n/);
+        decode_error("'true' expected")  if ($word =~ /^t/);
+        decode_error("'false' expected") if ($word =~ /^f/);
+        decode_error("malformed JSON string, neither array, object, number, string or atom");
+    }
+
+
+    sub number {
+        my $n    = '';
+        my $v;
+
+        # According to RFC4627, hex or oct digits are invalid.
+        if($ch eq '0'){
+            my $peek = substr($text,$at,1);
+            my $hex  = $peek =~ /[xX]/; # 0 or 1
+
+            if($hex){
+                decode_error("malformed number (leading zero must not be followed by another digit)");
+                ($n) = ( substr($text, $at+1) =~ /^([0-9a-fA-F]+)/);
+            }
+            else{ # oct
+                ($n) = ( substr($text, $at) =~ /^([0-7]+)/);
+                if (defined $n and length $n > 1) {
+                    decode_error("malformed number (leading zero must not be followed by another digit)");
+                }
+            }
+
+            if(defined $n and length($n)){
+                if (!$hex and length($n) == 1) {
+                   decode_error("malformed number (leading zero must not be followed by another digit)");
+                }
+                $at += length($n) + $hex;
+                next_chr;
+                return $hex ? hex($n) : oct($n);
+            }
+        }
+
+        if($ch eq '-'){
+            $n = '-';
+            next_chr;
+            if (!defined $ch or $ch !~ /\d/) {
+                decode_error("malformed number (no digits after initial minus)");
+            }
+        }
+
+        while(defined $ch and $ch =~ /\d/){
+            $n .= $ch;
+            next_chr;
+        }
+
+        if(defined $ch and $ch eq '.'){
+            $n .= '.';
+
+            next_chr;
+            if (!defined $ch or $ch !~ /\d/) {
+                decode_error("malformed number (no digits after decimal point)");
+            }
+            else {
+                $n .= $ch;
+            }
+
+            while(defined(next_chr) and $ch =~ /\d/){
+                $n .= $ch;
+            }
+        }
+
+        if(defined $ch and ($ch eq 'e' or $ch eq 'E')){
+            $n .= $ch;
+            next_chr;
+
+            if(defined($ch) and ($ch eq '+' or $ch eq '-')){
+                $n .= $ch;
+                next_chr;
+                if (!defined $ch or $ch =~ /\D/) {
+                    decode_error("malformed number (no digits after exp sign)");
+                }
+                $n .= $ch;
+            }
+            elsif(defined($ch) and $ch =~ /\d/){
+                $n .= $ch;
+            }
+            else {
+                decode_error("malformed number (no digits after exp sign)");
+            }
+
+            while(defined(next_chr) and $ch =~ /\d/){
+                $n .= $ch;
+            }
+
+        }
+
+        $v .= $n;
+
+        if ($v !~ /[.eE]/ and length $v > $max_intsize) {
+            if ($allow_bigint) { # from Adam Sussman
+                require Math::BigInt;
+                return Math::BigInt->new($v);
+            }
+            else {
+                return "$v";
+            }
+        }
+        elsif ($allow_bigint) {
+            require Math::BigFloat;
+            return Math::BigFloat->new($v);
+        }
+
+        return 0+$v;
+    }
+
+
+    sub is_valid_utf8 {
+
+        $utf8_len = $_[0] =~ /[\x00-\x7F]/  ? 1
+                  : $_[0] =~ /[\xC2-\xDF]/  ? 2
+                  : $_[0] =~ /[\xE0-\xEF]/  ? 3
+                  : $_[0] =~ /[\xF0-\xF4]/  ? 4
+                  : 0
+                  ;
+
+        return unless $utf8_len;
+
+        my $is_valid_utf8 = substr($text, $at - 1, $utf8_len);
+
+        return ( $is_valid_utf8 =~ /^(?:
+             [\x00-\x7F]
+            |[\xC2-\xDF][\x80-\xBF]
+            |[\xE0][\xA0-\xBF][\x80-\xBF]
+            |[\xE1-\xEC][\x80-\xBF][\x80-\xBF]
+            |[\xED][\x80-\x9F][\x80-\xBF]
+            |[\xEE-\xEF][\x80-\xBF][\x80-\xBF]
+            |[\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF]
+        )$/x )  ? $is_valid_utf8 : '';
+    }
+
+
+    sub decode_error {
+        my $error  = shift;
+        my $no_rep = shift;
+        my $str    = defined $text ? substr($text, $at) : '';
+        my $mess   = '';
+        my $type   = $] >= 5.008           ? 'U*'
+                   : $] <  5.006           ? 'C*'
+                   : utf8::is_utf8( $str ) ? 'U*' # 5.6
+                   : 'C*'
+                   ;
+
+        for my $c ( unpack( $type, $str ) ) { # emulate pv_uni_display() ?
+            $mess .=  $c == 0x07 ? '\a'
+                    : $c == 0x09 ? '\t'
+                    : $c == 0x0a ? '\n'
+                    : $c == 0x0d ? '\r'
+                    : $c == 0x0c ? '\f'
+                    : $c <  0x20 ? sprintf('\x{%x}', $c)
+                    : $c == 0x5c ? '\\\\'
+                    : $c <  0x80 ? chr($c)
+                    : sprintf('\x{%x}', $c)
+                    ;
+            if ( length $mess >= 20 ) {
+                $mess .= '...';
+                last;
+            }
+        }
+
+        unless ( length $mess ) {
+            $mess = '(end of string)';
+        }
+
+        Carp::croak (
+            $no_rep ? "$error" : "$error, at character offset $at (before \"$mess\")"
+        );
+
+    }
+
+
+    sub _json_object_hook {
+        my $o    = $_[0];
+        my @ks = keys %{$o};
+
+        if ( $cb_sk_object and @ks == 1 and exists $cb_sk_object->{ $ks[0] } and ref $cb_sk_object->{ $ks[0] } ) {
+            my @val = $cb_sk_object->{ $ks[0] }->( $o->{$ks[0]} );
+            if (@val == 1) {
+                return $val[0];
+            }
+        }
+
+        my @val = $cb_object->($o) if ($cb_object);
+        if (@val == 0 or @val > 1) {
+            return $o;
+        }
+        else {
+            return $val[0];
+        }
+    }
+
+
+    sub PP_decode_box {
+        {
+            text    => $text,
+            at      => $at,
+            ch      => $ch,
+            len     => $len,
+            depth   => $depth,
+            encoding      => $encoding,
+            is_valid_utf8 => $is_valid_utf8,
+        };
+    }
+
+} # PARSE
+
+
+sub _decode_surrogates { # from perlunicode
+    my $uni = 0x10000 + (hex($_[0]) - 0xD800) * 0x400 + (hex($_[1]) - 0xDC00);
+    my $un  = pack('U*', $uni);
+    utf8::encode( $un );
+    return $un;
+}
+
+
+sub _decode_unicode {
+    my $un = pack('U', hex shift);
+    utf8::encode( $un );
+    return $un;
+}
+
+#
+# Setup for various Perl versions (the code from JSON::PP58)
+#
+
+BEGIN {
+
+    unless ( defined &utf8::is_utf8 ) {
+       require Encode;
+       *utf8::is_utf8 = *Encode::is_utf8;
+    }
+
+    if ( $] >= 5.008 ) {
+        *JSON::PP::JSON_PP_encode_ascii      = \&_encode_ascii;
+        *JSON::PP::JSON_PP_encode_latin1     = \&_encode_latin1;
+        *JSON::PP::JSON_PP_decode_surrogates = \&_decode_surrogates;
+        *JSON::PP::JSON_PP_decode_unicode    = \&_decode_unicode;
+    }
+
+    if ($] >= 5.008 and $] < 5.008003) { # join() in 5.8.0 - 5.8.2 is broken.
+        package # hide from PAUSE
+          JSON::PP;
+        require subs;
+        subs->import('join');
+        eval q|
+            sub join {
+                return '' if (@_ < 2);
+                my $j   = shift;
+                my $str = shift;
+                for (@_) { $str .= $j . $_; }
+                return $str;
+            }
+        |;
+    }
+
+
+    sub JSON::PP::incr_parse {
+        local $Carp::CarpLevel = 1;
+        ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_parse( @_ );
+    }
+
+
+    sub JSON::PP::incr_skip {
+        ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_skip;
+    }
+
+
+    sub JSON::PP::incr_reset {
+        ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_reset;
+    }
+
+    eval q{
+        sub JSON::PP::incr_text : lvalue {
+            $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new;
+
+            if ( $_[0]->{_incr_parser}->{incr_parsing} ) {
+                Carp::croak("incr_text can not be called when the incremental parser already started parsing");
+            }
+            $_[0]->{_incr_parser}->{incr_text};
+        }
+    } if ( $] >= 5.006 );
+
+} # Setup for various Perl versions (the code from JSON::PP58)
+
+
+###############################
+# Utilities
+#
+
+BEGIN {
+    eval 'require Scalar::Util';
+    unless($@){
+        *JSON::PP::blessed = \&Scalar::Util::blessed;
+        *JSON::PP::reftype = \&Scalar::Util::reftype;
+        *JSON::PP::refaddr = \&Scalar::Util::refaddr;
+    }
+    else{ # This code is from Scalar::Util.
+        # warn $@;
+        eval 'sub UNIVERSAL::a_sub_not_likely_to_be_here { ref($_[0]) }';
+        *JSON::PP::blessed = sub {
+            local($@, $SIG{__DIE__}, $SIG{__WARN__});
+            ref($_[0]) ? eval { $_[0]->a_sub_not_likely_to_be_here } : undef;
+        };
+        my %tmap = qw(
+            B::NULL   SCALAR
+            B::HV     HASH
+            B::AV     ARRAY
+            B::CV     CODE
+            B::IO     IO
+            B::GV     GLOB
+            B::REGEXP REGEXP
+        );
+        *JSON::PP::reftype = sub {
+            my $r = shift;
+
+            return undef unless length(ref($r));
+
+            my $t = ref(B::svref_2object($r));
+
+            return
+                exists $tmap{$t} ? $tmap{$t}
+              : length(ref($$r)) ? 'REF'
+              :                    'SCALAR';
+        };
+        *JSON::PP::refaddr = sub {
+          return undef unless length(ref($_[0]));
+
+          my $addr;
+          if(defined(my $pkg = blessed($_[0]))) {
+            $addr .= bless $_[0], 'Scalar::Util::Fake';
+            bless $_[0], $pkg;
+          }
+          else {
+            $addr .= $_[0]
+          }
+
+          $addr =~ /0x(\w+)/;
+          local $^W;
+          #no warnings 'portable';
+          hex($1);
+        }
+    }
+}
+
+
+# shamelessly copied and modified from JSON::XS code.
+
+unless ( $INC{'JSON/PP.pm'} ) {
+    eval q|
+        package
+            JSON::PP::Boolean;
+
+        use overload (
+            "0+"     => sub { ${$_[0]} },
+            "++"     => sub { $_[0] = ${$_[0]} + 1 },
+            "--"     => sub { $_[0] = ${$_[0]} - 1 },
+            fallback => 1,
+        );
+    |;
+}
+
+$JSON::PP::true  = do { bless \(my $dummy = 1), "JSON::PP::Boolean" };
+$JSON::PP::false = do { bless \(my $dummy = 0), "JSON::PP::Boolean" };
+
+sub is_bool { defined $_[0] and UNIVERSAL::isa($_[0], "JSON::PP::Boolean"); }
+
+sub true  { $JSON::PP::true  }
+sub false { $JSON::PP::false }
+sub null  { undef; }
+
+###############################
+
+###############################
+
+package # hide from PAUSE
+  JSON::PP::IncrParser;
+
+use strict;
+
+use constant INCR_M_WS   => 0; # initial whitespace skipping
+use constant INCR_M_STR  => 1; # inside string
+use constant INCR_M_BS   => 2; # inside backslash
+use constant INCR_M_JSON => 3; # outside anything, count nesting
+use constant INCR_M_C0   => 4;
+use constant INCR_M_C1   => 5;
+
+use vars qw($VERSION);
+$VERSION = '1.01';
+
+my $unpack_format = $] < 5.006 ? 'C*' : 'U*';
+
+sub new {
+    my ( $class ) = @_;
+
+    bless {
+        incr_nest    => 0,
+        incr_text    => undef,
+        incr_parsing => 0,
+        incr_p       => 0,
+    }, $class;
+}
+
+
+sub incr_parse {
+    my ( $self, $coder, $text ) = @_;
+
+    $self->{incr_text} = '' unless ( defined $self->{incr_text} );
+
+    if ( defined $text ) {
+        if ( utf8::is_utf8( $text ) and !utf8::is_utf8( $self->{incr_text} ) ) {
+            utf8::upgrade( $self->{incr_text} ) ;
+            utf8::decode( $self->{incr_text} ) ;
+        }
+        $self->{incr_text} .= $text;
+    }
+
+
+    my $max_size = $coder->get_max_size;
+
+    if ( defined wantarray ) {
+
+        $self->{incr_mode} = INCR_M_WS unless defined $self->{incr_mode};
+
+        if ( wantarray ) {
+            my @ret;
+
+            $self->{incr_parsing} = 1;
+
+            do {
+                push @ret, $self->_incr_parse( $coder, $self->{incr_text} );
+
+                unless ( !$self->{incr_nest} and $self->{incr_mode} == INCR_M_JSON ) {
+                    $self->{incr_mode} = INCR_M_WS if $self->{incr_mode} != INCR_M_STR;
+                }
+
+            } until ( length $self->{incr_text} >= $self->{incr_p} );
+
+            $self->{incr_parsing} = 0;
+
+            return @ret;
+        }
+        else { # in scalar context
+            $self->{incr_parsing} = 1;
+            my $obj = $self->_incr_parse( $coder, $self->{incr_text} );
+            $self->{incr_parsing} = 0 if defined $obj; # pointed by Martin J. Evans
+            return $obj ? $obj : undef; # $obj is an empty string, parsing was completed.
+        }
+
+    }
+
+}
+
+
+sub _incr_parse {
+    my ( $self, $coder, $text, $skip ) = @_;
+    my $p = $self->{incr_p};
+    my $restore = $p;
+
+    my @obj;
+    my $len = length $text;
+
+    if ( $self->{incr_mode} == INCR_M_WS ) {
+        while ( $len > $p ) {
+            my $s = substr( $text, $p, 1 );
+            $p++ and next if ( 0x20 >= unpack($unpack_format, $s) );
+            $self->{incr_mode} = INCR_M_JSON;
+            last;
+       }
+    }
+
+    while ( $len > $p ) {
+        my $s = substr( $text, $p++, 1 );
+
+        if ( $s eq '"' ) {
+            if (substr( $text, $p - 2, 1 ) eq '\\' ) {
+                next;
+            }
+
+            if ( $self->{incr_mode} != INCR_M_STR  ) {
+                $self->{incr_mode} = INCR_M_STR;
+            }
+            else {
+                $self->{incr_mode} = INCR_M_JSON;
+                unless ( $self->{incr_nest} ) {
+                    last;
+                }
+            }
+        }
+
+        if ( $self->{incr_mode} == INCR_M_JSON ) {
+
+            if ( $s eq '[' or $s eq '{' ) {
+                if ( ++$self->{incr_nest} > $coder->get_max_depth ) {
+                    Carp::croak('json text or perl structure exceeds maximum nesting level (max_depth set too low?)');
+                }
+            }
+            elsif ( $s eq ']' or $s eq '}' ) {
+                last if ( --$self->{incr_nest} <= 0 );
+            }
+            elsif ( $s eq '#' ) {
+                while ( $len > $p ) {
+                    last if substr( $text, $p++, 1 ) eq "\n";
+                }
+            }
+
+        }
+
+    }
+
+    $self->{incr_p} = $p;
+
+    return if ( $self->{incr_mode} == INCR_M_STR and not $self->{incr_nest} );
+    return if ( $self->{incr_mode} == INCR_M_JSON and $self->{incr_nest} > 0 );
+
+    return '' unless ( length substr( $self->{incr_text}, 0, $p ) );
+
+    local $Carp::CarpLevel = 2;
+
+    $self->{incr_p} = $restore;
+    $self->{incr_c} = $p;
+
+    my ( $obj, $tail ) = $coder->PP_decode_json( substr( $self->{incr_text}, 0, $p ), 0x10000001 );
+
+    $self->{incr_text} = substr( $self->{incr_text}, $p );
+    $self->{incr_p} = 0;
+
+    return $obj || '';
+}
+
+
+sub incr_text {
+    if ( $_[0]->{incr_parsing} ) {
+        Carp::croak("incr_text can not be called when the incremental parser already started parsing");
+    }
+    $_[0]->{incr_text};
+}
+
+
+sub incr_skip {
+    my $self  = shift;
+    $self->{incr_text} = substr( $self->{incr_text}, $self->{incr_c} );
+    $self->{incr_p} = 0;
+}
+
+
+sub incr_reset {
+    my $self = shift;
+    $self->{incr_text}    = undef;
+    $self->{incr_p}       = 0;
+    $self->{incr_mode}    = 0;
+    $self->{incr_nest}    = 0;
+    $self->{incr_parsing} = 0;
+}
+
+###############################
+
+
+1;
+__END__
+=pod
+
+=head1 NAME
+
+JSON::PP - JSON::XS compatible pure-Perl module.
+
+=head1 SYNOPSIS
+
+ use JSON::PP;
+
+ # exported functions, they croak on error
+ # and expect/generate UTF-8
+
+ $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref;
+ $perl_hash_or_arrayref  = decode_json $utf8_encoded_json_text;
+
+ # OO-interface
+
+ $coder = JSON::PP->new->ascii->pretty->allow_nonref;
+ 
+ $json_text   = $json->encode( $perl_scalar );
+ $perl_scalar = $json->decode( $json_text );
+ 
+ $pretty_printed = $json->pretty->encode( $perl_scalar ); # pretty-printing
+ 
+ # Note that JSON version 2.0 and above will automatically use
+ # JSON::XS or JSON::PP, so you should be able to just:
+ 
+ use JSON;
+
+
+=head1 VERSION
+
+    2.27200
+
+L<JSON::XS> 2.27 (~2.30) compatible.
+
+=head1 DESCRIPTION
+
+This module is L<JSON::XS> compatible pure Perl module.
+(Perl 5.8 or later is recommended)
+
+JSON::XS is the fastest and most proper JSON module on CPAN.
+It is written by Marc Lehmann in C, so must be compiled and
+installed in the used environment.
+
+JSON::PP is a pure-Perl module and has compatibility to JSON::XS.
+
+
+=head2 FEATURES
+
+=over
+
+=item * correct unicode handling
+
+This module knows how to handle Unicode (depending on Perl version).
+
+See to L<JSON::XS/A FEW NOTES ON UNICODE AND PERL> and
+L<UNICODE HANDLING ON PERLS>.
+
+
+=item * round-trip integrity
+
+When you serialise a perl data structure using only data types
+supported by JSON and Perl, the deserialised data structure is
+identical on the Perl level. (e.g. the string "2.0" doesn't suddenly
+become "2" just because it looks like a number). There I<are> minor
+exceptions to this, read the MAPPING section below to learn about
+those.
+
+
+=item * strict checking of JSON correctness
+
+There is no guessing, no generating of illegal JSON texts by default,
+and only JSON is accepted as input by default (the latter is a
+security feature). But when some options are set, loose checking
+features are available.
+
+=back
+
+=head1 FUNCTIONAL INTERFACE
+
+Some documents are copied and modified from L<JSON::XS/FUNCTIONAL INTERFACE>.
+
+=head2 encode_json
+
+    $json_text = encode_json $perl_scalar
+
+Converts the given Perl data structure to a UTF-8 encoded, binary string.
+
+This function call is functionally identical to:
+
+    $json_text = JSON::PP->new->utf8->encode($perl_scalar)
+
+=head2 decode_json
+
+    $perl_scalar = decode_json $json_text
+
+The opposite of C<encode_json>: expects an UTF-8 (binary) string and tries
+to parse that as an UTF-8 encoded JSON text, returning the resulting
+reference.
+
+This function call is functionally identical to:
+
+    $perl_scalar = JSON::PP->new->utf8->decode($json_text)
+
+=head2 JSON::PP::is_bool
+
+    $is_boolean = JSON::PP::is_bool($scalar)
+
+Returns true if the passed scalar represents either JSON::PP::true or
+JSON::PP::false, two constants that act like C<1> and C<0> respectively
+and are also used to represent JSON C<true> and C<false> in Perl strings.
+
+=head2 JSON::PP::true
+
+Returns JSON true value which is blessed object.
+It C<isa> JSON::PP::Boolean object.
+
+=head2 JSON::PP::false
+
+Returns JSON false value which is blessed object.
+It C<isa> JSON::PP::Boolean object.
+
+=head2 JSON::PP::null
+
+Returns C<undef>.
+
+See L<MAPPING>, below, for more information on how JSON values are mapped to
+Perl.
+
+
+=head1 HOW DO I DECODE A DATA FROM OUTER AND ENCODE TO OUTER
+
+This section supposes that your perl version is 5.8 or later.
+
+If you know a JSON text from an outer world - a network, a file content, and so on,
+is encoded in UTF-8, you should use C<decode_json> or C<JSON> module object
+with C<utf8> enable. And the decoded result will contain UNICODE characters.
+
+  # from network
+  my $json        = JSON::PP->new->utf8;
+  my $json_text   = CGI->new->param( 'json_data' );
+  my $perl_scalar = $json->decode( $json_text );
+  
+  # from file content
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  $json_text   = <$fh>;
+  $perl_scalar = decode_json( $json_text );
+
+If an outer data is not encoded in UTF-8, firstly you should C<decode> it.
+
+  use Encode;
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  my $encoding = 'cp932';
+  my $unicode_json_text = decode( $encoding, <$fh> ); # UNICODE
+  
+  # or you can write the below code.
+  #
+  # open( my $fh, "<:encoding($encoding)", 'json.data' );
+  # $unicode_json_text = <$fh>;
+
+In this case, C<$unicode_json_text> is of course UNICODE string.
+So you B<cannot> use C<decode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable.
+
+  $perl_scalar = $json->utf8(0)->decode( $unicode_json_text );
+
+Or C<encode 'utf8'> and C<decode_json>:
+
+  $perl_scalar = decode_json( encode( 'utf8', $unicode_json_text ) );
+  # this way is not efficient.
+
+And now, you want to convert your C<$perl_scalar> into JSON data and
+send it to an outer world - a network or a file content, and so on.
+
+Your data usually contains UNICODE strings and you want the converted data to be encoded
+in UTF-8, you should use C<encode_json> or C<JSON> module object with C<utf8> enable.
+
+  print encode_json( $perl_scalar ); # to a network? file? or display?
+  # or
+  print $json->utf8->encode( $perl_scalar );
+
+If C<$perl_scalar> does not contain UNICODE but C<$encoding>-encoded strings
+for some reason, then its characters are regarded as B<latin1> for perl
+(because it does not concern with your $encoding).
+You B<cannot> use C<encode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable.
+Note that the resulted text is a UNICODE string but no problem to print it.
+
+  # $perl_scalar contains $encoding encoded string values
+  $unicode_json_text = $json->utf8(0)->encode( $perl_scalar );
+  # $unicode_json_text consists of characters less than 0x100
+  print $unicode_json_text;
+
+Or C<decode $encoding> all string values and C<encode_json>:
+
+  $perl_scalar->{ foo } = decode( $encoding, $perl_scalar->{ foo } );
+  # ... do it to each string values, then encode_json
+  $json_text = encode_json( $perl_scalar );
+
+This method is a proper way but probably not efficient.
+
+See to L<Encode>, L<perluniintro>.
+
+
+=head1 METHODS
+
+Basically, check to L<JSON> or L<JSON::XS>.
+
+=head2 new
+
+    $json = JSON::PP->new
+
+Returns a new JSON::PP object that can be used to de/encode JSON
+strings.
+
+All boolean flags described below are by default I<disabled>.
+
+The mutators for flags all return the JSON object again and thus calls can
+be chained:
+
+   my $json = JSON::PP->new->utf8->space_after->encode({a => [1,2]})
+   => {"a": [1, 2]}
+
+=head2 ascii
+
+    $json = $json->ascii([$enable])
+    
+    $enabled = $json->get_ascii
+
+If $enable is true (or missing), then the encode method will not generate characters outside
+the code range 0..127. Any Unicode characters outside that range will be escaped using either
+a single \uXXXX or a double \uHHHH\uLLLLL escape sequence, as per RFC4627.
+(See to L<JSON::XS/OBJECT-ORIENTED INTERFACE>).
+
+In Perl 5.005, there is no character having high value (more than 255).
+See to L<UNICODE HANDLING ON PERLS>.
+
+If $enable is false, then the encode method will not escape Unicode characters unless
+required by the JSON syntax or other flags. This results in a faster and more compact format.
+
+  JSON::PP->new->ascii(1)->encode([chr 0x10401])
+  => ["\ud801\udc01"]
+
+=head2 latin1
+
+    $json = $json->latin1([$enable])
+    
+    $enabled = $json->get_latin1
+
+If $enable is true (or missing), then the encode method will encode the resulting JSON
+text as latin1 (or iso-8859-1), escaping any characters outside the code range 0..255.
+
+If $enable is false, then the encode method will not escape Unicode characters
+unless required by the JSON syntax or other flags.
+
+  JSON::XS->new->latin1->encode (["\x{89}\x{abc}"]
+  => ["\x{89}\\u0abc"]    # (perl syntax, U+abc escaped, U+89 not)
+
+See to L<UNICODE HANDLING ON PERLS>.
+
+=head2 utf8
+
+    $json = $json->utf8([$enable])
+    
+    $enabled = $json->get_utf8
+
+If $enable is true (or missing), then the encode method will encode the JSON result
+into UTF-8, as required by many protocols, while the decode method expects to be handled
+an UTF-8-encoded string. Please note that UTF-8-encoded strings do not contain any
+characters outside the range 0..255, they are thus useful for bytewise/binary I/O.
+
+(In Perl 5.005, any character outside the range 0..255 does not exist.
+See to L<UNICODE HANDLING ON PERLS>.)
+
+In future versions, enabling this option might enable autodetection of the UTF-16 and UTF-32
+encoding families, as described in RFC4627.
+
+If $enable is false, then the encode method will return the JSON string as a (non-encoded)
+Unicode string, while decode expects thus a Unicode string. Any decoding or encoding
+(e.g. to UTF-8 or UTF-16) needs to be done yourself, e.g. using the Encode module.
+
+Example, output UTF-16BE-encoded JSON:
+
+  use Encode;
+  $jsontext = encode "UTF-16BE", JSON::PP->new->encode ($object);
+
+Example, decode UTF-32LE-encoded JSON:
+
+  use Encode;
+  $object = JSON::PP->new->decode (decode "UTF-32LE", $jsontext);
+
+
+=head2 pretty
+
+    $json = $json->pretty([$enable])
+
+This enables (or disables) all of the C<indent>, C<space_before> and
+C<space_after> flags in one call to generate the most readable
+(or most compact) form possible.
+
+Equivalent to:
+
+   $json->indent->space_before->space_after
+
+=head2 indent
+
+    $json = $json->indent([$enable])
+    
+    $enabled = $json->get_indent
+
+The default indent space length is three.
+You can use C<indent_length> to change the length.
+
+=head2 space_before
+
+    $json = $json->space_before([$enable])
+    
+    $enabled = $json->get_space_before
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space before the C<:> separating keys from values in JSON objects.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before enabled, space_after and indent disabled:
+
+   {"key" :"value"}
+
+=head2 space_after
+
+    $json = $json->space_after([$enable])
+    
+    $enabled = $json->get_space_after
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space after the C<:> separating keys from values in JSON objects
+and extra whitespace after the C<,> separating key-value pairs and array
+members.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before and indent disabled, space_after enabled:
+
+   {"key": "value"}
+
+=head2 relaxed
+
+    $json = $json->relaxed([$enable])
+    
+    $enabled = $json->get_relaxed
+
+If C<$enable> is true (or missing), then C<decode> will accept some
+extensions to normal JSON syntax (see below). C<encode> will not be
+affected in anyway. I<Be aware that this option makes you accept invalid
+JSON texts as if they were valid!>. I suggest only to use this option to
+parse application-specific files written by humans (configuration files,
+resource files etc.)
+
+If C<$enable> is false (the default), then C<decode> will only accept
+valid JSON texts.
+
+Currently accepted extensions are:
+
+=over 4
+
+=item * list items can have an end-comma
+
+JSON I<separates> array elements and key-value pairs with commas. This
+can be annoying if you write JSON texts manually and want to be able to
+quickly append elements, so this extension accepts comma at the end of
+such items not just between them:
+
+   [
+      1,
+      2, <- this comma not normally allowed
+   ]
+   {
+      "k1": "v1",
+      "k2": "v2", <- this comma not normally allowed
+   }
+
+=item * shell-style '#'-comments
+
+Whenever JSON allows whitespace, shell-style comments are additionally
+allowed. They are terminated by the first carriage-return or line-feed
+character, after which more white-space and comments are allowed.
+
+  [
+     1, # this comment not allowed in JSON
+        # neither this one...
+  ]
+
+=back
+
+=head2 canonical
+
+    $json = $json->canonical([$enable])
+    
+    $enabled = $json->get_canonical
+
+If C<$enable> is true (or missing), then the C<encode> method will output JSON objects
+by sorting their keys. This is adding a comparatively high overhead.
+
+If C<$enable> is false, then the C<encode> method will output key-value
+pairs in the order Perl stores them (which will likely change between runs
+of the same script).
+
+This option is useful if you want the same data structure to be encoded as
+the same JSON text (given the same overall settings). If it is disabled,
+the same hash might be encoded differently even if contains the same data,
+as key-value pairs have no inherent ordering in Perl.
+
+This setting has no effect when decoding JSON texts.
+
+If you want your own sorting routine, you can give a code reference
+or a subroutine name to C<sort_by>. See to C<JSON::PP OWN METHODS>.
+
+=head2 allow_nonref
+
+    $json = $json->allow_nonref([$enable])
+    
+    $enabled = $json->get_allow_nonref
+
+If C<$enable> is true (or missing), then the C<encode> method can convert a
+non-reference into its corresponding string, number or null JSON value,
+which is an extension to RFC4627. Likewise, C<decode> will accept those JSON
+values instead of croaking.
+
+If C<$enable> is false, then the C<encode> method will croak if it isn't
+passed an arrayref or hashref, as JSON texts must either be an object
+or array. Likewise, C<decode> will croak if given something that is not a
+JSON object or array.
+
+   JSON::PP->new->allow_nonref->encode ("Hello, World!")
+   => "Hello, World!"
+
+=head2 allow_unknown
+
+    $json = $json->allow_unknown ([$enable])
+    
+    $enabled = $json->get_allow_unknown
+
+If $enable is true (or missing), then "encode" will *not* throw an
+exception when it encounters values it cannot represent in JSON (for
+example, filehandles) but instead will encode a JSON "null" value.
+Note that blessed objects are not included here and are handled
+separately by c<allow_nonref>.
+
+If $enable is false (the default), then "encode" will throw an
+exception when it encounters anything it cannot encode as JSON.
+
+This option does not affect "decode" in any way, and it is
+recommended to leave it off unless you know your communications
+partner.
+
+=head2 allow_blessed
+
+    $json = $json->allow_blessed([$enable])
+    
+    $enabled = $json->get_allow_blessed
+
+If C<$enable> is true (or missing), then the C<encode> method will not
+barf when it encounters a blessed reference. Instead, the value of the
+B<convert_blessed> option will decide whether C<null> (C<convert_blessed>
+disabled or no C<TO_JSON> method found) or a representation of the
+object (C<convert_blessed> enabled and C<TO_JSON> method found) is being
+encoded. Has no effect on C<decode>.
+
+If C<$enable> is false (the default), then C<encode> will throw an
+exception when it encounters a blessed object.
+
+=head2 convert_blessed
+
+    $json = $json->convert_blessed([$enable])
+    
+    $enabled = $json->get_convert_blessed
+
+If C<$enable> is true (or missing), then C<encode>, upon encountering a
+blessed object, will check for the availability of the C<TO_JSON> method
+on the object's class. If found, it will be called in scalar context
+and the resulting scalar will be encoded instead of the object. If no
+C<TO_JSON> method is found, the value of C<allow_blessed> will decide what
+to do.
+
+The C<TO_JSON> method may safely call die if it wants. If C<TO_JSON>
+returns other blessed objects, those will be handled in the same
+way. C<TO_JSON> must take care of not causing an endless recursion cycle
+(== crash) in this case. The name of C<TO_JSON> was chosen because other
+methods called by the Perl core (== not by the user of the object) are
+usually in upper case letters and to avoid collisions with the C<to_json>
+function or method.
+
+This setting does not yet influence C<decode> in any way.
+
+If C<$enable> is false, then the C<allow_blessed> setting will decide what
+to do when a blessed object is found.
+
+=head2 filter_json_object
+
+    $json = $json->filter_json_object([$coderef])
+
+When C<$coderef> is specified, it will be called from C<decode> each
+time it decodes a JSON object. The only argument passed to the coderef
+is a reference to the newly-created hash. If the code references returns
+a single scalar (which need not be a reference), this value
+(i.e. a copy of that scalar to avoid aliasing) is inserted into the
+deserialised data structure. If it returns an empty list
+(NOTE: I<not> C<undef>, which is a valid scalar), the original deserialised
+hash will be inserted. This setting can slow down decoding considerably.
+
+When C<$coderef> is omitted or undefined, any existing callback will
+be removed and C<decode> will not change the deserialised hash in any
+way.
+
+Example, convert all JSON objects into the integer 5:
+
+   my $js = JSON::PP->new->filter_json_object (sub { 5 });
+   # returns [5]
+   $js->decode ('[{}]'); # the given subroutine takes a hash reference.
+   # throw an exception because allow_nonref is not enabled
+   # so a lone 5 is not allowed.
+   $js->decode ('{"a":1, "b":2}');
+
+=head2 filter_json_single_key_object
+
+    $json = $json->filter_json_single_key_object($key [=> $coderef])
+
+Works remotely similar to C<filter_json_object>, but is only called for
+JSON objects having a single key named C<$key>.
+
+This C<$coderef> is called before the one specified via
+C<filter_json_object>, if any. It gets passed the single value in the JSON
+object. If it returns a single value, it will be inserted into the data
+structure. If it returns nothing (not even C<undef> but the empty list),
+the callback from C<filter_json_object> will be called next, as if no
+single-key callback were specified.
+
+If C<$coderef> is omitted or undefined, the corresponding callback will be
+disabled. There can only ever be one callback for a given key.
+
+As this callback gets called less often then the C<filter_json_object>
+one, decoding speed will not usually suffer as much. Therefore, single-key
+objects make excellent targets to serialise Perl objects into, especially
+as single-key JSON objects are as close to the type-tagged value concept
+as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not
+support this in any way, so you need to make sure your data never looks
+like a serialised Perl hash.
+
+Typical names for the single object key are C<__class_whatever__>, or
+C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even
+things like C<__class_md5sum(classname)__>, to reduce the risk of clashing
+with real hashes.
+
+Example, decode JSON objects of the form C<< { "__widget__" => <id> } >>
+into the corresponding C<< $WIDGET{<id>} >> object:
+
+   # return whatever is in $WIDGET{5}:
+   JSON::PP
+      ->new
+      ->filter_json_single_key_object (__widget__ => sub {
+            $WIDGET{ $_[0] }
+         })
+      ->decode ('{"__widget__": 5')
+
+   # this can be used with a TO_JSON method in some "widget" class
+   # for serialisation to json:
+   sub WidgetBase::TO_JSON {
+      my ($self) = @_;
+
+      unless ($self->{id}) {
+         $self->{id} = ..get..some..id..;
+         $WIDGET{$self->{id}} = $self;
+      }
+
+      { __widget__ => $self->{id} }
+   }
+
+=head2 shrink
+
+    $json = $json->shrink([$enable])
+    
+    $enabled = $json->get_shrink
+
+In JSON::XS, this flag resizes strings generated by either
+C<encode> or C<decode> to their minimum size possible.
+It will also try to downgrade any strings to octet-form if possible.
+
+In JSON::PP, it is noop about resizing strings but tries
+C<utf8::downgrade> to the returned string by C<encode>.
+See to L<utf8>.
+
+See to L<JSON::XS/OBJECT-ORIENTED INTERFACE>
+
+=head2 max_depth
+
+    $json = $json->max_depth([$maximum_nesting_depth])
+    
+    $max_depth = $json->get_max_depth
+
+Sets the maximum nesting level (default C<512>) accepted while encoding
+or decoding. If a higher nesting level is detected in JSON text or a Perl
+data structure, then the encoder and decoder will stop and croak at that
+point.
+
+Nesting level is defined by number of hash- or arrayrefs that the encoder
+needs to traverse to reach a given point or the number of C<{> or C<[>
+characters without their matching closing parenthesis crossed to reach a
+given character in a string.
+
+If no argument is given, the highest possible setting will be used, which
+is rarely useful.
+
+See L<JSON::XS/SSECURITY CONSIDERATIONS> for more info on why this is useful.
+
+When a large value (100 or more) was set and it de/encodes a deep nested object/text,
+it may raise a warning 'Deep recursion on subroutine' at the perl runtime phase.
+
+=head2 max_size
+
+    $json = $json->max_size([$maximum_string_size])
+    
+    $max_size = $json->get_max_size
+
+Set the maximum length a JSON text may have (in bytes) where decoding is
+being attempted. The default is C<0>, meaning no limit. When C<decode>
+is called on a string that is longer then this many bytes, it will not
+attempt to decode the string but throw an exception. This setting has no
+effect on C<encode> (yet).
+
+If no argument is given, the limit check will be deactivated (same as when
+C<0> is specified).
+
+See L<JSON::XS/SECURITY CONSIDERATIONS> for more info on why this is useful.
+
+=head2 encode
+
+    $json_text = $json->encode($perl_scalar)
+
+Converts the given Perl data structure (a simple scalar or a reference
+to a hash or array) to its JSON representation. Simple scalars will be
+converted into JSON string or number sequences, while references to arrays
+become JSON arrays and references to hashes become JSON objects. Undefined
+Perl values (e.g. C<undef>) become JSON C<null> values.
+References to the integers C<0> and C<1> are converted into C<true> and C<false>.
+
+=head2 decode
+
+    $perl_scalar = $json->decode($json_text)
+
+The opposite of C<encode>: expects a JSON text and tries to parse it,
+returning the resulting simple scalar or reference. Croaks on error.
+
+JSON numbers and strings become simple Perl scalars. JSON arrays become
+Perl arrayrefs and JSON objects become Perl hashrefs. C<true> becomes
+C<1> (C<JSON::true>), C<false> becomes C<0> (C<JSON::false>) and
+C<null> becomes C<undef>.
+
+=head2 decode_prefix
+
+    ($perl_scalar, $characters) = $json->decode_prefix($json_text)
+
+This works like the C<decode> method, but instead of raising an exception
+when there is trailing garbage after the first JSON object, it will
+silently stop parsing there and return the number of characters consumed
+so far.
+
+   JSON->new->decode_prefix ("[1] the tail")
+   => ([], 3)
+
+=head1 INCREMENTAL PARSING
+
+Most of this section are copied and modified from L<JSON::XS/INCREMENTAL PARSING>.
+
+In some cases, there is the need for incremental parsing of JSON texts.
+This module does allow you to parse a JSON stream incrementally.
+It does so by accumulating text until it has a full JSON object, which
+it then can decode. This process is similar to using C<decode_prefix>
+to see if a full JSON object is available, but is much more efficient
+(and can be implemented with a minimum of method calls).
+
+This module will only attempt to parse the JSON text once it is sure it
+has enough text to get a decisive result, using a very simple but
+truly incremental parser. This means that it sometimes won't stop as
+early as the full parser, for example, it doesn't detect parenthesis
+mismatches. The only thing it guarantees is that it starts decoding as
+soon as a syntactically valid JSON text has been seen. This means you need
+to set resource limits (e.g. C<max_size>) to ensure the parser will stop
+parsing in the presence if syntax errors.
+
+The following methods implement this incremental parser.
+
+=head2 incr_parse
+
+    $json->incr_parse( [$string] ) # void context
+    
+    $obj_or_undef = $json->incr_parse( [$string] ) # scalar context
+    
+    @obj_or_empty = $json->incr_parse( [$string] ) # list context
+
+This is the central parsing function. It can both append new text and
+extract objects from the stream accumulated so far (both of these
+functions are optional).
+
+If C<$string> is given, then this string is appended to the already
+existing JSON fragment stored in the C<$json> object.
+
+After that, if the function is called in void context, it will simply
+return without doing anything further. This can be used to add more text
+in as many chunks as you want.
+
+If the method is called in scalar context, then it will try to extract
+exactly I<one> JSON object. If that is successful, it will return this
+object, otherwise it will return C<undef>. If there is a parse error,
+this method will croak just as C<decode> would do (one can then use
+C<incr_skip> to skip the erroneous part). This is the most common way of
+using the method.
+
+And finally, in list context, it will try to extract as many objects
+from the stream as it can find and return them, or the empty list
+otherwise. For this to work, there must be no separators between the JSON
+objects or arrays, instead they must be concatenated back-to-back. If
+an error occurs, an exception will be raised as in the scalar context
+case. Note that in this case, any previously-parsed JSON texts will be
+lost.
+
+Example: Parse some JSON arrays/objects in a given string and return them.
+
+    my @objs = JSON->new->incr_parse ("[5][7][1,2]");
+
+=head2 incr_text
+
+    $lvalue_string = $json->incr_text
+
+This method returns the currently stored JSON fragment as an lvalue, that
+is, you can manipulate it. This I<only> works when a preceding call to
+C<incr_parse> in I<scalar context> successfully returned an object. Under
+all other circumstances you must not call this function (I mean it.
+although in simple tests it might actually work, it I<will> fail under
+real world conditions). As a special exception, you can also call this
+method before having parsed anything.
+
+This function is useful in two cases: a) finding the trailing text after a
+JSON object or b) parsing multiple JSON objects separated by non-JSON text
+(such as commas).
+
+    $json->incr_text =~ s/\s*,\s*//;
+
+In Perl 5.005, C<lvalue> attribute is not available.
+You must write codes like the below:
+
+    $string = $json->incr_text;
+    $string =~ s/\s*,\s*//;
+    $json->incr_text( $string );
+
+=head2 incr_skip
+
+    $json->incr_skip
+
+This will reset the state of the incremental parser and will remove the
+parsed text from the input buffer. This is useful after C<incr_parse>
+died, in which case the input buffer and incremental parser state is left
+unchanged, to skip the text parsed so far and to reset the parse state.
+
+=head2 incr_reset
+
+    $json->incr_reset
+
+This completely resets the incremental parser, that is, after this call,
+it will be as if the parser had never parsed anything.
+
+This is useful if you want to repeatedly parse JSON objects and want to
+ignore any trailing data, which means you have to reset the parser after
+each successful decode.
+
+See to L<JSON::XS/INCREMENTAL PARSING> for examples.
+
+
+=head1 JSON::PP OWN METHODS
+
+=head2 allow_singlequote
+
+    $json = $json->allow_singlequote([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+JSON strings quoted by single quotations that are invalid JSON
+format.
+
+    $json->allow_singlequote->decode({"foo":'bar'});
+    $json->allow_singlequote->decode({'foo':"bar"});
+    $json->allow_singlequote->decode({'foo':'bar'});
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+
+=head2 allow_barekey
+
+    $json = $json->allow_barekey([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+bare keys of JSON object that are invalid JSON format.
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+    $json->allow_barekey->decode('{foo:"bar"}');
+
+=head2 allow_bignum
+
+    $json = $json->allow_bignum([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will convert
+the big integer Perl cannot handle as integer into a L<Math::BigInt>
+object and convert a floating number (any) into a L<Math::BigFloat>.
+
+On the contrary, C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers with C<allow_blessed> enable.
+
+   $json->allow_nonref->allow_blessed->allow_bignum;
+   $bigfloat = $json->decode('2.000000000000000000000000001');
+   print $json->encode($bigfloat);
+   # => 2.000000000000000000000000001
+
+See to L<JSON::XS/MAPPING> about the normal conversion of JSON number.
+
+=head2 loose
+
+    $json = $json->loose([$enable])
+
+The unescaped [\x00-\x1f\x22\x2f\x5c] strings are invalid in JSON strings
+and the module doesn't allow to C<decode> to these (except for \x2f).
+If C<$enable> is true (or missing), then C<decode>  will accept these
+unescaped strings.
+
+    $json->loose->decode(qq|["abc
+                                   def"]|);
+
+See L<JSON::XS/SSECURITY CONSIDERATIONS>.
+
+=head2 escape_slash
+
+    $json = $json->escape_slash([$enable])
+
+According to JSON Grammar, I<slash> (U+002F) is escaped. But default
+JSON::PP (as same as JSON::XS) encodes strings without escaping slash.
+
+If C<$enable> is true (or missing), then C<encode> will escape slashes.
+
+=head2 indent_length
+
+    $json = $json->indent_length($length)
+
+JSON::XS indent space length is 3 and cannot be changed.
+JSON::PP set the indent space length with the given $length.
+The default is 3. The acceptable range is 0 to 15.
+
+=head2 sort_by
+
+    $json = $json->sort_by($function_name)
+    $json = $json->sort_by($subroutine_ref)
+
+If $function_name or $subroutine_ref are set, its sort routine are used
+in encoding JSON objects.
+
+   $js = $pc->sort_by(sub { $JSON::PP::a cmp $JSON::PP::b })->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   $js = $pc->sort_by('own_sort')->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   sub JSON::PP::own_sort { $JSON::PP::a cmp $JSON::PP::b }
+
+As the sorting routine runs in the JSON::PP scope, the given
+subroutine name and the special variables C<$a>, C<$b> will begin
+'JSON::PP::'.
+
+If $integer is set, then the effect is same as C<canonical> on.
+
+=head1 INTERNAL
+
+For developers.
+
+=over
+
+=item PP_encode_box
+
+Returns
+
+        {
+            depth        => $depth,
+            indent_count => $indent_count,
+        }
+
+
+=item PP_decode_box
+
+Returns
+
+        {
+            text    => $text,
+            at      => $at,
+            ch      => $ch,
+            len     => $len,
+            depth   => $depth,
+            encoding      => $encoding,
+            is_valid_utf8 => $is_valid_utf8,
+        };
+
+=back
+
+=head1 MAPPING
+
+This section is copied from JSON::XS and modified to C<JSON::PP>.
+JSON::XS and JSON::PP mapping mechanisms are almost equivalent.
+
+See to L<JSON::XS/MAPPING>.
+
+=head2 JSON -> PERL
+
+=over 4
+
+=item object
+
+A JSON object becomes a reference to a hash in Perl. No ordering of object
+keys is preserved (JSON does not preserver object key ordering itself).
+
+=item array
+
+A JSON array becomes a reference to an array in Perl.
+
+=item string
+
+A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON
+are represented by the same codepoints in the Perl string, so no manual
+decoding is necessary.
+
+=item number
+
+A JSON number becomes either an integer, numeric (floating point) or
+string scalar in perl, depending on its range and any fractional parts. On
+the Perl level, there is no difference between those as Perl handles all
+the conversion details, but an integer may take slightly less memory and
+might represent more values exactly than floating point numbers.
+
+If the number consists of digits only, C<JSON> will try to represent
+it as an integer value. If that fails, it will try to represent it as
+a numeric (floating point) value if that is possible without loss of
+precision. Otherwise it will preserve the number as a string value (in
+which case you lose roundtripping ability, as the JSON number will be
+re-encoded to a JSON string).
+
+Numbers containing a fractional or exponential part will always be
+represented as numeric (floating point) values, possibly at a loss of
+precision (in which case you might lose perfect roundtripping ability, but
+the JSON number will still be re-encoded as a JSON number).
+
+Note that precision is not accuracy - binary floating point values cannot
+represent most decimal fractions exactly, and when converting from and to
+floating point, C<JSON> only guarantees precision up to but not including
+the least significant bit.
+
+When C<allow_bignum> is enable, the big integers 
+and the numeric can be optionally converted into L<Math::BigInt> and
+L<Math::BigFloat> objects.
+
+=item true, false
+
+These JSON atoms become C<JSON::PP::true> and C<JSON::PP::false>,
+respectively. They are overloaded to act almost exactly like the numbers
+C<1> and C<0>. You can check whether a scalar is a JSON boolean by using
+the C<JSON::is_bool> function.
+
+   print JSON::PP::true . "\n";
+    => true
+   print JSON::PP::true + 1;
+    => 1
+
+   ok(JSON::true eq  '1');
+   ok(JSON::true == 1);
+
+C<JSON> will install these missing overloading features to the backend modules.
+
+
+=item null
+
+A JSON null atom becomes C<undef> in Perl.
+
+C<JSON::PP::null> returns C<undef>.
+
+=back
+
+
+=head2 PERL -> JSON
+
+The mapping from Perl to JSON is slightly more difficult, as Perl is a
+truly typeless language, so we can only guess which JSON type is meant by
+a Perl value.
+
+=over 4
+
+=item hash references
+
+Perl hash references become JSON objects. As there is no inherent ordering
+in hash keys (or JSON objects), they will usually be encoded in a
+pseudo-random order that can change between runs of the same program but
+stays generally the same within a single run of a program. C<JSON>
+optionally sort the hash keys (determined by the I<canonical> flag), so
+the same data structure will serialise to the same JSON text (given same
+settings and version of JSON::XS), but this incurs a runtime overhead
+and is only rarely useful, e.g. when you want to compare some JSON text
+against another for equality.
+
+
+=item array references
+
+Perl array references become JSON arrays.
+
+=item other references
+
+Other unblessed references are generally not allowed and will cause an
+exception to be thrown, except for references to the integers C<0> and
+C<1>, which get turned into C<false> and C<true> atoms in JSON. You can
+also use C<JSON::false> and C<JSON::true> to improve readability.
+
+   to_json [\0,JSON::PP::true]      # yields [false,true]
+
+=item JSON::PP::true, JSON::PP::false, JSON::PP::null
+
+These special values become JSON true and JSON false values,
+respectively. You can also use C<\1> and C<\0> directly if you want.
+
+JSON::PP::null returns C<undef>.
+
+=item blessed objects
+
+Blessed objects are not directly representable in JSON. See the
+C<allow_blessed> and C<convert_blessed> methods on various options on
+how to deal with this: basically, you can choose between throwing an
+exception, encoding the reference as if it weren't blessed, or provide
+your own serialiser method.
+
+See to L<convert_blessed>.
+
+=item simple scalars
+
+Simple Perl scalars (any scalar that is not a reference) are the most
+difficult objects to encode: JSON::XS and JSON::PP will encode undefined scalars as
+JSON C<null> values, scalars that have last been used in a string context
+before encoding as JSON strings, and anything else as number value:
+
+   # dump as number
+   encode_json [2]                      # yields [2]
+   encode_json [-3.0e17]                # yields [-3e+17]
+   my $value = 5; encode_json [$value]  # yields [5]
+
+   # used as string, so dump as string
+   print $value;
+   encode_json [$value]                 # yields ["5"]
+
+   # undef becomes null
+   encode_json [undef]                  # yields [null]
+
+You can force the type to be a string by stringifying it:
+
+   my $x = 3.1; # some variable containing a number
+   "$x";        # stringified
+   $x .= "";    # another, more awkward way to stringify
+   print $x;    # perl does it for you, too, quite often
+
+You can force the type to be a number by numifying it:
+
+   my $x = "3"; # some variable containing a string
+   $x += 0;     # numify it, ensuring it will be dumped as a number
+   $x *= 1;     # same thing, the choice is yours.
+
+You can not currently force the type in other, less obscure, ways.
+
+Note that numerical precision has the same meaning as under Perl (so
+binary to decimal conversion follows the same rules as in Perl, which
+can differ to other languages). Also, your perl interpreter might expose
+extensions to the floating point numbers of your platform, such as
+infinities or NaN's - these cannot be represented in JSON, and it is an
+error to pass those in.
+
+=item Big Number
+
+When C<allow_bignum> is enable, 
+C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers.
+
+
+=back
+
+=head1 UNICODE HANDLING ON PERLS
+
+If you do not know about Unicode on Perl well,
+please check L<JSON::XS/A FEW NOTES ON UNICODE AND PERL>.
+
+=head2 Perl 5.8 and later
+
+Perl can handle Unicode and the JSON::PP de/encode methods also work properly.
+
+    $json->allow_nonref->encode(chr hex 3042);
+    $json->allow_nonref->encode(chr hex 12345);
+
+Returns C<"\u3042"> and C<"\ud808\udf45"> respectively.
+
+    $json->allow_nonref->decode('"\u3042"');
+    $json->allow_nonref->decode('"\ud808\udf45"');
+
+Returns UTF-8 encoded strings with UTF8 flag, regarded as C<U+3042> and C<U+12345>.
+
+Note that the versions from Perl 5.8.0 to 5.8.2, Perl built-in C<join> was broken,
+so JSON::PP wraps the C<join> with a subroutine. Thus JSON::PP works slow in the versions.
+
+
+=head2 Perl 5.6
+
+Perl can handle Unicode and the JSON::PP de/encode methods also work.
+
+=head2 Perl 5.005
+
+Perl 5.005 is a byte semantics world -- all strings are sequences of bytes.
+That means the unicode handling is not available.
+
+In encoding,
+
+    $json->allow_nonref->encode(chr hex 3042);  # hex 3042 is 12354.
+    $json->allow_nonref->encode(chr hex 12345); # hex 12345 is 74565.
+
+Returns C<B> and C<E>, as C<chr> takes a value more than 255, it treats
+as C<$value % 256>, so the above codes are equivalent to :
+
+    $json->allow_nonref->encode(chr 66);
+    $json->allow_nonref->encode(chr 69);
+
+In decoding,
+
+    $json->decode('"\u00e3\u0081\u0082"');
+
+The returned is a byte sequence C<0xE3 0x81 0x82> for UTF-8 encoded
+japanese character (C<HIRAGANA LETTER A>).
+And if it is represented in Unicode code point, C<U+3042>.
+
+Next, 
+
+    $json->decode('"\u3042"');
+
+We ordinary expect the returned value is a Unicode character C<U+3042>.
+But here is 5.005 world. This is C<0xE3 0x81 0x82>.
+
+    $json->decode('"\ud808\udf45"');
+
+This is not a character C<U+12345> but bytes - C<0xf0 0x92 0x8d 0x85>.
+
+
+=head1 TODO
+
+=over
+
+=item speed
+
+=item memory saving
+
+=back
+
+
+=head1 SEE ALSO
+
+Most of the document are copied and modified from JSON::XS doc.
+
+L<JSON::XS>
+
+RFC4627 (L<http://www.ietf.org/rfc/rfc4627.txt>)
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2007-2012 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut

uroman/lib/JSON/backportPP/Boolean.pm

@@ -0,0 +1,27 @@
+=head1 NAME
+
+JSON::PP::Boolean - dummy module providing JSON::PP::Boolean
+
+=head1 SYNOPSIS
+
+ # do not "use" yourself
+
+=head1 DESCRIPTION
+
+This module exists only to provide overload resolution for Storable
+and similar modules. See L<JSON::PP> for more info about this class.
+
+=cut
+
+use JSON::backportPP ();
+use strict;
+
+1;
+
+=head1 AUTHOR
+
+This idea is from L<JSON::XS::Boolean> written by
+Marc Lehmann <schmorp[at]schmorp.de>
+
+=cut
+

uroman/lib/JSON/backportPP/Compat5005.pm

@@ -0,0 +1,131 @@
+package # This is JSON::backportPP
+    JSON::backportPP5005;
+
+use 5.005;
+use strict;
+
+my @properties;
+
+$JSON::PP5005::VERSION = '1.10';
+
+BEGIN {
+
+    sub utf8::is_utf8 {
+        0; # It is considered that UTF8 flag off for Perl 5.005.
+    }
+
+    sub utf8::upgrade {
+    }
+
+    sub utf8::downgrade {
+        1; # must always return true.
+    }
+
+    sub utf8::encode  {
+    }
+
+    sub utf8::decode {
+    }
+
+    *JSON::PP::JSON_PP_encode_ascii      = \&_encode_ascii;
+    *JSON::PP::JSON_PP_encode_latin1     = \&_encode_latin1;
+    *JSON::PP::JSON_PP_decode_surrogates = \&_decode_surrogates;
+    *JSON::PP::JSON_PP_decode_unicode    = \&_decode_unicode;
+
+    # missing in B module.
+    sub B::SVp_IOK () { 0x01000000; }
+    sub B::SVp_NOK () { 0x02000000; }
+    sub B::SVp_POK () { 0x04000000; }
+
+    $INC{'bytes.pm'} = 1; # dummy
+}
+
+
+
+sub _encode_ascii {
+    join('', map { $_ <= 127 ? chr($_) : sprintf('\u%04x', $_) } unpack('C*', $_[0]) );
+}
+
+
+sub _encode_latin1 {
+    join('', map { chr($_) } unpack('C*', $_[0]) );
+}
+
+
+sub _decode_surrogates { # from http://homepage1.nifty.com/nomenclator/unicode/ucs_utf.htm
+    my $uni = 0x10000 + (hex($_[0]) - 0xD800) * 0x400 + (hex($_[1]) - 0xDC00); # from perlunicode
+    my $bit = unpack('B32', pack('N', $uni));
+
+    if ( $bit =~ /^00000000000(...)(......)(......)(......)$/ ) {
+        my ($w, $x, $y, $z) = ($1, $2, $3, $4);
+        return pack('B*', sprintf('11110%s10%s10%s10%s', $w, $x, $y, $z));
+    }
+    else {
+        Carp::croak("Invalid surrogate pair");
+    }
+}
+
+
+sub _decode_unicode {
+    my ($u) = @_;
+    my ($utf8bit);
+
+    if ( $u =~ /^00([89a-f][0-9a-f])$/i ) { # 0x80-0xff
+         return pack( 'H2', $1 );
+    }
+
+    my $bit = unpack("B*", pack("H*", $u));
+
+    if ( $bit =~ /^00000(.....)(......)$/ ) {
+        $utf8bit = sprintf('110%s10%s', $1, $2);
+    }
+    elsif ( $bit =~ /^(....)(......)(......)$/ ) {
+        $utf8bit = sprintf('1110%s10%s10%s', $1, $2, $3);
+    }
+    else {
+        Carp::croak("Invalid escaped unicode");
+    }
+
+    return pack('B*', $utf8bit);
+}
+
+
+sub JSON::PP::incr_text {
+    $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new;
+
+    if ( $_[0]->{_incr_parser}->{incr_parsing} ) {
+        Carp::croak("incr_text can not be called when the incremental parser already started parsing");
+    }
+
+    $_[0]->{_incr_parser}->{incr_text} = $_[1] if ( @_ > 1 );
+    $_[0]->{_incr_parser}->{incr_text};
+}
+
+
+1;
+__END__
+
+=pod
+
+=head1 NAME
+
+JSON::PP5005 - Helper module in using JSON::PP in Perl 5.005
+
+=head1 DESCRIPTION
+
+JSON::PP calls internally.
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2007-2012 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut
+

uroman/lib/JSON/backportPP/Compat5006.pm

@@ -0,0 +1,173 @@
+package # This is JSON::backportPP
+    JSON::backportPP56;
+
+use 5.006;
+use strict;
+
+my @properties;
+
+$JSON::PP56::VERSION = '1.08';
+
+BEGIN {
+
+    sub utf8::is_utf8 {
+        my $len =  length $_[0]; # char length
+        {
+            use bytes; #  byte length;
+            return $len != length $_[0]; # if !=, UTF8-flagged on.
+        }
+    }
+
+
+    sub utf8::upgrade {
+        ; # noop;
+    }
+
+
+    sub utf8::downgrade ($;$) {
+        return 1 unless ( utf8::is_utf8( $_[0] ) );
+
+        if ( _is_valid_utf8( $_[0] ) ) {
+            my $downgrade;
+            for my $c ( unpack( "U*", $_[0] ) ) {
+                if ( $c < 256 ) {
+                    $downgrade .= pack("C", $c);
+                }
+                else {
+                    $downgrade .= pack("U", $c);
+                }
+            }
+            $_[0] = $downgrade;
+            return 1;
+        }
+        else {
+            Carp::croak("Wide character in subroutine entry") unless ( $_[1] );
+            0;
+        }
+    }
+
+
+    sub utf8::encode ($) { # UTF8 flag off
+        if ( utf8::is_utf8( $_[0] ) ) {
+            $_[0] = pack( "C*", unpack( "C*", $_[0] ) );
+        }
+        else {
+            $_[0] = pack( "U*", unpack( "C*", $_[0] ) );
+            $_[0] = pack( "C*", unpack( "C*", $_[0] ) );
+        }
+    }
+
+
+    sub utf8::decode ($) { # UTF8 flag on
+        if ( _is_valid_utf8( $_[0] ) ) {
+            utf8::downgrade( $_[0] );
+            $_[0] = pack( "U*", unpack( "U*", $_[0] ) );
+        }
+    }
+
+
+    *JSON::PP::JSON_PP_encode_ascii      = \&_encode_ascii;
+    *JSON::PP::JSON_PP_encode_latin1     = \&_encode_latin1;
+    *JSON::PP::JSON_PP_decode_surrogates = \&JSON::PP::_decode_surrogates;
+    *JSON::PP::JSON_PP_decode_unicode    = \&JSON::PP::_decode_unicode;
+
+    unless ( defined &B::SVp_NOK ) { # missing in B module.
+        eval q{ sub B::SVp_NOK () { 0x02000000; } };
+    }
+
+}
+
+
+
+sub _encode_ascii {
+    join('',
+        map {
+            $_ <= 127 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', JSON::PP::_encode_surrogates($_));
+        } _unpack_emu($_[0])
+    );
+}
+
+
+sub _encode_latin1 {
+    join('',
+        map {
+            $_ <= 255 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', JSON::PP::_encode_surrogates($_));
+        } _unpack_emu($_[0])
+    );
+}
+
+
+sub _unpack_emu { # for Perl 5.6 unpack warnings
+    return   !utf8::is_utf8($_[0]) ? unpack('C*', $_[0]) 
+           : _is_valid_utf8($_[0]) ? unpack('U*', $_[0])
+           : unpack('C*', $_[0]);
+}
+
+
+sub _is_valid_utf8 {
+    my $str = $_[0];
+    my $is_utf8;
+
+    while ($str =~ /(?:
+          (
+             [\x00-\x7F]
+            |[\xC2-\xDF][\x80-\xBF]
+            |[\xE0][\xA0-\xBF][\x80-\xBF]
+            |[\xE1-\xEC][\x80-\xBF][\x80-\xBF]
+            |[\xED][\x80-\x9F][\x80-\xBF]
+            |[\xEE-\xEF][\x80-\xBF][\x80-\xBF]
+            |[\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF]
+          )
+        | (.)
+    )/xg)
+    {
+        if (defined $1) {
+            $is_utf8 = 1 if (!defined $is_utf8);
+        }
+        else {
+            $is_utf8 = 0 if (!defined $is_utf8);
+            if ($is_utf8) { # eventually, not utf8
+                return;
+            }
+        }
+    }
+
+    return $is_utf8;
+}
+
+
+1;
+__END__
+
+=pod
+
+=head1 NAME
+
+JSON::PP56 - Helper module in using JSON::PP in Perl 5.6
+
+=head1 DESCRIPTION
+
+JSON::PP calls internally.
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2007-2012 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut
+

uroman/lib/NLP/Chinese.pm

@@ -0,0 +1,239 @@
+################################################################
+#                                                              #
+# Chinese                                                      #
+#                                                              #
+################################################################
+
+package NLP::Chinese;
+
+$utf8 = NLP::UTF8;
+%empty_ht = ();
+
+sub read_chinese_tonal_pinyin_files {
+   local($caller, *ht, @filenames) = @_;
+
+   $n_kHanyuPinlu = 0;
+   $n_kXHC1983 = 0;
+   $n_kHanyuPinyin = 0;
+   $n_kMandarin = 0;
+   $n_cedict = 0;
+   $n_simple_pinyin = 0;
+
+   foreach $filename (@filenames) {
+      if ($filename =~ /unihan/i) {
+	 my $line_number = 0;
+         if (open(IN, $filename)) {
+            while (<IN>) {
+	       $line_number++;
+               next if /^#/;
+               s/\s*$//;
+               if (($u, $type, $value) = split(/\t/, $_)) {
+                  if ($type =~ /^(kHanyuPinlu|kXHC1983|kHanyuPinyin|kMandarin)$/) {
+	             $u = $util->trim($u);
+	             $type = $util->trim($type);
+	             $value = $util->trim($value);
+                     $f = $utf8->unicode_string2string($u);
+
+                     if ($type eq "kHanyuPinlu") {
+	                $value =~ s/\(.*?\)//g;
+			$value = $util->trim($value);
+                        $translit = $caller->number_to_accent_tone($value);
+	                $ht{"kHanyuPinlu"}->{$f} = $translit;
+	                $n_kHanyuPinlu++;
+                     } elsif ($type eq "kXHC1983") {
+	                @translits = ($value =~ /:(\S+)/g);
+	                $translit = join(" ", @translits);
+	                $ht{"kXHC1983"}->{$f} = $translit;
+	                $n_kXHC1983++;
+                     } elsif ($type eq "kHanyuPinyin") {
+	                $value =~ s/^.*://;
+	                $value =~ s/,/ /g;
+	                $ht{"kHanyuPinyin"}->{$f} = $value;
+	                $n_kHanyuPinyin++;
+                     } elsif ($type eq "kMandarin") {
+			$ht{"kMandarin"}->{$f} = $value;
+			$n_kMandarin++;
+                     }
+                  }
+               }
+            }
+            close(IN);
+            print "Read in $n_kHanyuPinlu kHanyuPinlu, $n_kXHC1983 n_kXHC1983, $n_kHanyuPinyin n_kHanyuPinyin $n_kMandarin n_kMandarin\n";
+         } else {
+	    print STDERR "Can't open $filename\n";
+	 }
+      } elsif ($filename =~ /cedict/i) {
+         if (open(IN, $filename)) {
+	    my $line_number = 0;
+            while (<IN>) {
+	       $line_number++;
+               next if /^#/;
+               s/\s*$//;
+               if (($f, $translit) = ($_ =~ /^\S+\s+(\S+)\s+\[([^\[\]]+)\]/)) {
+                  $translit = $utf8->extended_lower_case($translit);
+                  $translit = $caller->number_to_accent_tone($translit);
+                  $translit =~ s/\s//g;
+		  if ($old_translit = $ht{"cedict"}->{$f}) {
+                     # $ht{CONFLICT}->{("DUPLICATE " . $f)} = "CEDICT($f): $old_translit\nCEDICT($f): $translit (duplicate)\n" unless $translit eq $old_translit;
+                     $ht{"cedicts"}->{$f} = join(" ", $ht{"cedicts"}->{$f}, $translit) unless $old_translit eq $translit;
+		  } else {
+                     $ht{"cedict"}->{$f} = $translit;
+                     $ht{"cedicts"}->{$f} = $translit;
+		  }
+                  $n_cedict++;
+	       }
+            }
+            close(IN);
+            # print "Read in $n_cedict n_cedict\n";
+         } else {
+	    print STDERR "Can't open $filename";
+         }
+      } elsif ($filename =~ /chinese_to_pinyin/i) {
+	 if (open(IN, $filename)) {
+	    my $line_number = 0;
+	    while (<IN>) {
+	       $line_number++;
+	       next if /^#/;
+	       if (($f, $translit) = ($_ =~ /^(\S+)\t(\S+)\s*$/)) {
+		  $ht{"simple_pinyin"}->{$f} = $translit; 
+		  $n_simple_pinyin++;
+	       }
+	    }
+	    close(IN);
+	    # print "Read in $n_simple_pinyin n_simple_pinyin\n";
+	 } else {
+	    print STDERR "Can't open $filename";
+	 }
+      } else {
+	 print STDERR "Don't know what to do with file $filename (in read_chinese_tonal_pinyin_files)\n";
+      }
+   }
+}
+
+sub tonal_pinyin {
+   local($caller, $s, *ht, $gloss) = @_;
+
+   return $result if defined($result = $ht{COMBINED}->{$s});
+
+   $cedict_pinyin = $ht{"cedict"}->{$s} || "";
+   $cedicts_pinyin = $ht{"cedicts"}->{$s} || "";
+   $unihan_pinyin = "";
+   @characters = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   foreach $c (@characters) {
+      if ($pinyin = $ht{"simple_pinyin"}->{$c}) {
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"kHanyuPinlu"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"kXHC1983"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"kHanyuPinyin"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"cedicts"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      # middle dot, katakana middle dot, multiplication sign
+      } elsif ($c =~ /^(\xC2\xB7|\xE3\x83\xBB|\xC3\x97)$/) {
+	 $unihan_pinyin .= $c;
+      # ASCII
+      } elsif ($c =~ /^([\x21-\x7E])$/) {
+	 $unihan_pinyin .= $c;
+      } else {
+	 $unihan_pinyin .= "?";
+	 $hex = $utf8->utf8_to_hex($c);
+	 $unicode = uc $utf8->utf8_to_4hex_unicode($c);
+	 # print STDERR "Tonal pinyin: Unknown character $c ($hex/U+$unicode) -> ?\n";
+      }
+   }
+   $pinyin_title = "";
+   if (($#characters >= 1) && $cedicts_pinyin) {
+      foreach $pinyin (split(/\s+/, $cedicts_pinyin)) {
+	 $pinyin_title .= "$s $pinyin (CEDICT)\n";
+      }
+      $pinyin_title .= "\n";
+   }
+   foreach $c (@characters) {
+      my %local_ht = ();
+      @pinyins = ();
+      foreach $type (("kHanyuPinlu", "kXHC1983", "kHanyuPinyin", "cedicts")) {
+	 if ($pinyin_s = $ht{$type}->{$c}) {
+	    foreach $pinyin (split(/\s+/, $pinyin_s)) {
+	       push(@pinyins, $pinyin) unless $util->member($pinyin, @pinyins);
+	       $type2 = ($type eq "cedicts") ? "CEDICT" : $type;
+	       $local_ht{$pinyin} = ($local_ht{$pinyin}) ? join(", ", $local_ht{$pinyin}, $type2) : $type2;
+	    }
+	 }
+      }
+      foreach $pinyin (@pinyins) {
+	 $type_s = $local_ht{$pinyin};
+	 $pinyin_title .= "$c $pinyin ($type_s)\n";
+      }
+   }
+   $pinyin_title =~ s/\n$//;
+   $pinyin_title =~ s/\n/&#xA;/g;
+   $unihan_pinyin = "" if $unihan_pinyin =~ /^\?+$/;
+   if (($#characters >= 1) && $cedict_pinyin && $unihan_pinyin && ($unihan_pinyin ne $cedict_pinyin)) {
+      $log = "Gloss($s): $gloss\nCEdict($s): $cedicts_pinyin\nUnihan($s): $unihan_pinyin\n";
+      foreach $type (("kHanyuPinlu", "kXHC1983", "kHanyuPinyin")) {
+	 $log_line = "$type($s): ";
+	 foreach $c (@characters) {
+	    $pinyin = $ht{$type}->{$c} || "";
+	    if ($pinyin =~ / /) {
+	       $log_line .= "($pinyin)";
+	    } elsif ($pinyin) {
+	       $log_line .= $pinyin;
+	    } else {
+	       $log_line .= "?";
+	    }
+	 }
+	 $log .= "$log_line\n";
+      }
+      $ht{CONFLICT}->{$s} = $log;
+   }
+   $result = $unihan_pinyin || $cedict_pinyin;
+   $result = $cedict_pinyin if ($#characters > 0) && $cedict_pinyin;
+   $ht{COMBINED}->{$s} = $result;
+   $ht{PINYIN_TITLE}->{$s} = $pinyin_title;
+   return $result;
+}
+
+%number_to_accent_tone_ht = (
+   "a1", "\xC4\x81", "a2", "\xC3\xA1", "a3", "\xC7\x8E", "a4", "\xC3\xA0",
+   "e1", "\xC4\x93", "e2", "\xC3\xA9", "e3", "\xC4\x9B", "e4", "\xC3\xA8",
+   "i1", "\xC4\xAB", "i2", "\xC3\xAD", "i3", "\xC7\x90", "i4", "\xC3\xAC",
+   "o1", "\xC5\x8D", "o2", "\xC3\xB3", "o3", "\xC7\x92", "o4", "\xC3\xB2",
+   "u1", "\xC5\xAB", "u2", "\xC3\xBA", "u3", "\xC7\x94", "u4", "\xC3\xB9",
+   "u:1","\xC7\x96", "u:2","\xC7\x98", "u:3","\xC7\x9A", "u:4","\xC7\x9C",
+   "\xC3\xBC1","\xC7\x96","\xC3\xBC2","\xC7\x98","\xC3\xBC3","\xC7\x9A","\xC3\xBC4","\xC7\x9C"
+);
+
+sub number_to_accent_tone {
+   local($caller, $s) = @_;
+
+   my $result = "";
+   while (($pre,$alpha,$tone_number,$rest) = ($s =~ /^(.*?)((?:[a-z]|u:|\xC3\xBC)+)([1-5])(.*)$/i)) {
+      if ($tone_number eq "5") {
+         $result .= "$pre$alpha";
+      } elsif ((($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)([ae])(.*)$/))
+       || (($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)(o)(u.*)$/))
+       || (($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)(u:|[iou]|\xC3\xBC)([^aeiou]*)$/))) {
+         $result .= "$pre$pre_acc" . ($number_to_accent_tone_ht{($acc_letter . $tone_number)} || ($acc_letter . $tone_number)) . $post_acc;
+      } else {
+         $result .= "$pre$alpha$tone_number";
+      }
+      $s = $rest;
+   }
+   $result .= $s;
+   $result =~ s/u:/\xC3\xBC/g;
+   return $result;
+}
+
+sub string_contains_utf8_cjk_unified_ideograph_p {
+   local($caller, $s) = @_;
+
+   return ($s =~ /([\xE4-\xE9]|\xE3[\x90-\xBF]|\xF0[\xA0-\xAC])/);
+}
+
+1;

uroman/lib/NLP/Romanizer.pm

@@ -0,0 +1,2020 @@
+################################################################
+#                                                              #
+# Romanizer                                                    #
+#                                                              #
+################################################################
+
+package NLP::Romanizer;
+
+use NLP::Chinese;
+use NLP::UTF8;
+use NLP::utilities;
+use JSON;
+$utf8 = NLP::UTF8;
+$util = NLP::utilities;
+$chinesePM = NLP::Chinese;
+
+my $verbosePM = 0;
+%empty_ht = ();
+
+my $braille_capital_letter_indicator = "\xE2\xA0\xA0";
+my $braille_number_indicator = "\xE2\xA0\xBC";
+my $braille_decimal_point = "\xE2\xA0\xA8";
+my $braille_comma = "\xE2\xA0\x82";
+my $braille_solidus = "\xE2\xA0\x8C";
+my $braille_numeric_space = "\xE2\xA0\x90";
+my $braille_letter_indicator = "\xE2\xA0\xB0";
+my $braille_period = "\xE2\xA0\xB2";
+
+sub new {
+   local($caller) = @_;
+
+   my $object = {};
+   my $class = ref( $caller ) || $caller;
+   bless($object, $class);
+   return $object;
+}
+
+sub load_unicode_data {
+   local($this, *ht, $filename) = @_;
+   # ../../data/UnicodeData.txt
+
+   $n = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+	 if (($unicode_value, $char_name, $general_category, $canon_comb_classes, $bidir_category, $char_decomp_mapping, $decimal_digit_value, $digit_value, $numeric_value, $mirrored, $unicode_1_0_name, $comment_field, $uc_mapping, $lc_mapping, $title_case_mapping) = split(";", $_)) {
+            $utf8_code = $utf8->unicode_hex_string2string($unicode_value);
+	    $ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name;
+	    $ht{UTF_NAME_TO_UNICODE}->{$char_name} = $unicode_value;
+	    $ht{UTF_NAME_TO_CODE}->{$char_name} = $utf8_code;
+	    $ht{UTF_TO_CAT}->{$utf8_code} = $general_category;
+	    $ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric_value unless $numeric_value eq "";
+	    $n++;
+	 }
+      }
+      close(IN);
+      # print STDERR "Loaded $n entries from $filename\n";
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub load_unicode_overwrite_romanization {
+   local($this, *ht, $filename) = @_;
+   # ../../data/UnicodeDataOverwrite.txt
+
+   $n = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+	 next if /^#/;
+         $unicode_value = $util->slot_value_in_double_colon_del_list($_, "u");
+         $romanization = $util->slot_value_in_double_colon_del_list($_, "r");
+         $numeric = $util->slot_value_in_double_colon_del_list($_, "num");
+         $picture = $util->slot_value_in_double_colon_del_list($_, "pic");
+	 $syllable_info = $util->slot_value_in_double_colon_del_list($_, "syllable-info");
+	 $tone_mark = $util->slot_value_in_double_colon_del_list($_, "tone-mark");
+	 $char_name = $util->slot_value_in_double_colon_del_list($_, "name");
+	 $entry_processed_p = 0;
+         $utf8_code = $utf8->unicode_hex_string2string($unicode_value);
+	 if ($unicode_value) {
+	    $ht{UTF_TO_CHAR_ROMANIZATION}->{$utf8_code} = $romanization if $romanization;
+	    $ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric if defined($numeric) && ($numeric ne "");
+	    $ht{UTF_TO_PICTURE_DESCR}->{$utf8_code} = $picture if $picture;
+	    $ht{UTF_TO_SYLLABLE_INFO}->{$utf8_code} = $syllable_info if $syllable_info;
+	    $ht{UTF_TO_TONE_MARK}->{$utf8_code} = $tone_mark if $tone_mark;
+	    $ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name if $char_name;
+	    $entry_processed_p = 1 if $romanization || $numeric || $picture || $syllable_info || $tone_mark;
+	 }
+	 $n++ if $entry_processed_p;
+      }
+      close(IN);
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub load_script_data {
+   local($this, *ht, $filename) = @_;
+   # ../../data/Scripts.txt
+
+   $n = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+         next unless $script_name = $util->slot_value_in_double_colon_del_list($_, "script-name");
+         $abugida_default_vowel_s = $util->slot_value_in_double_colon_del_list($_, "abugida-default-vowel");
+         $alt_script_name_s = $util->slot_value_in_double_colon_del_list($_, "alt-script-name");
+         $language_s = $util->slot_value_in_double_colon_del_list($_, "language");
+         $direction = $util->slot_value_in_double_colon_del_list($_, "direction"); # right-to-left
+         $font_family_s = $util->slot_value_in_double_colon_del_list($_, "font-family");
+         $ht{SCRIPT_P}->{$script_name} = 1;
+	 $ht{SCRIPT_NORM}->{(uc $script_name)} = $script_name;
+         $ht{DIRECTION}->{$script_name} = $direction if $direction;
+	 foreach $language (split(/,\s*/, $language_s)) {
+	    $ht{SCRIPT_LANGUAGE}->{$script_name}->{$language} = 1;
+	    $ht{LANGUAGE_SCRIPT}->{$language}->{$script_name} = 1;
+	 }
+	 foreach $alt_script_name (split(/,\s*/, $alt_script_name_s)) {
+	    $ht{SCRIPT_NORM}->{$alt_script_name} = $script_name;
+	    $ht{SCRIPT_NORM}->{(uc $alt_script_name)} = $script_name;
+	 }
+	 foreach $abugida_default_vowel (split(/,\s*/, $abugida_default_vowel_s)) {
+	    $ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$script_name}->{$abugida_default_vowel} = 1 if $abugida_default_vowel;
+	 }
+	 foreach $font_family (split(/,\s*/, $font_family_s)) {
+	    $ht{SCRIPT_FONT}->{$script_name}->{$font_family} = 1 if $font_family;
+	 }
+	 $n++;
+      }
+      close(IN);
+      # print STDERR "Loaded $n entries from $filename\n";
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub unicode_hangul_romanization {
+   local($this, $s, $pass_through_p) = @_;
+
+   $pass_through_p = 0 unless defined($pass_through_p);
+   @leads = split(/\s+/, "g gg n d dd r m b bb s ss - j jj c k t p h");
+   # @vowels = split(/\s+/, "a ae ya yai e ei ye yei o oa oai oi yo u ue uei ui yu w wi i");
+   @vowels = split(/\s+/, "a ae ya yae eo e yeo ye o wa wai oe yo u weo we wi yu eu yi i");
+   @tails = split(/\s+/, "- g gg gs n nj nh d l lg lm lb ls lt lp lh m b bs s ss ng j c k t p h");
+   $result = "";
+   @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   foreach $char (@chars) {
+      $unicode = $utf8->utf8_to_unicode($char);
+      if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
+	 $code = $unicode - 0xAC00;
+	 $lead_index = int($code / (28*21));
+	 $vowel_index = int($code/28) % 21;
+	 $tail_index = $code % 28;
+	 $rom = $leads[$lead_index] . $vowels[$vowel_index] . $tails[$tail_index];
+	 $rom =~ s/-//g;
+	 $result .= $rom;
+      } elsif ($pass_through_p) {
+	 $result .= $char;
+      }
+   }
+   return $result;
+}
+
+sub listify_comma_sep_string {
+   local($this, $s) = @_;
+
+   @result_list = ();
+   return @result_list unless $s =~ /\S/;
+   $s = $util->trim2($s);
+   my $elem;
+
+   while (($elem, $rest) = ($s =~ /^("(?:\\"|[^"])*"|'(?:\\'|[^'])*'|[^"', ]+),\s*(.*)$/)) {
+      push(@result_list, $util->dequote_string($elem));
+      $s = $rest;
+   }
+   push(@result_list, $util->dequote_string($s)) if $s =~ /\S/;
+
+   return @result_list;
+}
+
+sub braille_string_p {
+   local($this, $s) = @_;
+
+   return ($s =~ /^(\xE2[\xA0-\xA3][\x80-\xBF])+$/);
+}
+
+sub register_word_boundary_info {
+   local($this, *ht, $lang_code, $utf8_source_string, $utf8_target_string, $use_only_for_whole_word_p, 
+	 $use_only_at_start_of_word_p, $use_only_at_end_of_word_p, 
+	 $dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p) = @_;
+
+   if ($use_only_for_whole_word_p) {
+      if ($lang_code) {
+         $ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{USE_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($use_only_at_start_of_word_p) {
+      if ($lang_code) {
+         $ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{USE_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($use_only_at_end_of_word_p) {
+      if ($lang_code) {
+         $ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{USE_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($dont_use_at_start_of_word_p) {
+      if ($lang_code) {
+         $ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{DONT_USE_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($dont_use_at_end_of_word_p) {
+      if ($lang_code) {
+         $ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{DONT_USE_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+}
+
+sub load_romanization_table {
+   local($this, *ht, $filename) = @_;
+   # ../../data/romanization-table.txt
+
+   $n = 0;
+   $line_number = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+         $line_number++;
+	 next if /^#/;
+	 if ($_ =~ /^::preserve\s/) {
+	    $from_unicode = $util->slot_value_in_double_colon_del_list($_, "from");
+	    $to_unicode = $util->slot_value_in_double_colon_del_list($_, "to");
+	    if ($from_unicode =~ /^(?:U\+|\\u)[0-9A-F]{4,}$/i) {
+	       $from_unicode =~ s/^(?:U\+|\\u)//;
+	       $from_code_point = hex($from_unicode);
+	    } else {
+	       $from_code_point = "";
+	    }
+	    if ($to_unicode =~ /^(?:U\+|\\u)[0-9A-F]{4,}$/i) {
+	       $to_unicode =~ s/^(?:U\+|\\u)//;
+	       $to_code_point = hex($to_unicode);
+	    } else {
+	       $to_code_point = $from_code_point;
+	    }
+	    if ($from_code_point ne "") {
+	       # print STDERR "Preserve code-points $from_unicode--$to_unicode = $from_code_point--$to_code_point\n";
+	       foreach $code_point (($from_code_point .. $to_code_point)) {
+	          $utf8_string = $utf8->unicode2string($code_point);
+                  $ht{UTF_CHAR_MAPPING}->{$utf8_string}->{$utf8_string} = 1;
+	       }
+	       $n++;
+	    }
+	    next;
+	 }
+         $utf8_source_string = $util->slot_value_in_double_colon_del_list($_, "s");
+         $utf8_target_string = $util->slot_value_in_double_colon_del_list($_, "t");
+         $utf8_alt_target_string_s = $util->slot_value_in_double_colon_del_list($_, "t-alt");
+         $use_alt_in_pointed_p = ($_ =~ /::use-alt-in-pointed\b/);
+         $use_only_for_whole_word_p = ($_ =~ /::use-only-for-whole-word\b/);
+         $use_only_at_start_of_word_p = ($_ =~ /::use-only-at-start-of-word\b/);
+         $use_only_at_end_of_word_p = ($_ =~ /::use-only-at-end-of-word\b/);
+         $dont_use_at_start_of_word_p = ($_ =~ /::dont-use-at-start-of-word\b/);
+         $dont_use_at_end_of_word_p = ($_ =~ /::dont-use-at-end-of-word\b/);
+	 $use_only_in_lower_case_enviroment_p = ($_ =~ /::use-only-in-lower-case-enviroment\b/);
+	 $word_external_punctuation_p = ($_ =~ /::word-external-punctuation\b/);
+	 $utf8_source_string =~ s/\s*$//;
+	 $utf8_target_string =~ s/\s*$//;
+	 $utf8_alt_target_string_s =~ s/\s*$//;
+	 $utf8_target_string =~ s/^"(.*)"$/$1/;
+	 $utf8_target_string =~ s/^'(.*)'$/$1/;
+	 @utf8_alt_targets = $this->listify_comma_sep_string($utf8_alt_target_string_s);
+         $numeric = $util->slot_value_in_double_colon_del_list($_, "num");
+	 $numeric =~ s/\s*$//;
+         $annotation = $util->slot_value_in_double_colon_del_list($_, "annotation");
+	 $annotation =~ s/\s*$//;
+         $lang_code = $util->slot_value_in_double_colon_del_list($_, "lcode");
+         $prob = $util->slot_value_in_double_colon_del_list($_, "p") || 1;
+	 unless (($utf8_target_string eq "") && ($numeric =~ /\d/)) {
+	    if ($lang_code) {
+               $ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	    } else {
+               $ht{UTF_CHAR_MAPPING}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	    }
+	    if ($word_external_punctuation_p) {
+	       if ($lang_code) {
+                  $ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	       } else {
+                  $ht{WORD_EXTERNAL_PUNCTUATION}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	       }
+	    }
+            if ($this->braille_string_p($utf8_source_string)) {
+	       if (($utf8_target_string =~ /^[a-z]+$/)
+	        && (! ($utf8_source_string =~ /^$braille_capital_letter_indicator/))) {
+	          my $uc_utf8_source_string = "$braille_capital_letter_indicator$utf8_source_string";
+	          my $uc_utf8_target_string = ucfirst $utf8_target_string;
+	          if ($lang_code) {
+                     $ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
+	          } else {
+                     $ht{UTF_CHAR_MAPPING}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
+	          }
+                  $this->register_word_boundary_info(*ht, $lang_code, $uc_utf8_source_string, $uc_utf8_target_string,
+                            $use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p, 
+	                    $dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
+	       }
+	       if (($utf8_target_string =~ /^[0-9]$/)
+	        && ($utf8_source_string =~ /^$braille_number_indicator./)) {
+	          my $core_number_char = $utf8_source_string;
+		  $core_number_char =~ s/$braille_number_indicator//;
+	          $ht{BRAILLE_TO_DIGIT}->{$core_number_char} = $utf8_target_string;
+	       }
+	    }
+	 }
+	 if ($use_only_in_lower_case_enviroment_p) {
+	    if ($lang_code) {
+	       $ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+	    } else {
+	       $ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT}->{$utf8_source_string}->{$utf8_target_string} = 1;
+	    }
+	 }
+         $this->register_word_boundary_info(*ht, $lang_code, $utf8_source_string, $utf8_target_string, 
+                   $use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p, 
+	           $dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
+	 foreach $utf8_alt_target (@utf8_alt_targets) {
+	    if ($lang_code) {
+               $ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
+	       $ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
+	    } else {
+               $ht{UTF_CHAR_ALT_MAPPING}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
+	       $ht{USE_ALT_IN_POINTED}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
+	    }
+	    if ($use_only_for_whole_word_p) {
+	       if ($lang_code) {
+	          $ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       } else {
+	          $ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       }
+	    }
+	    if ($use_only_at_start_of_word_p) {
+	       if ($lang_code) {
+	          $ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       } else {
+	          $ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       }
+	    }
+	    if ($use_only_at_end_of_word_p) {
+	       if ($lang_code) {
+	          $ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       } else {
+	          $ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       }
+	    }
+	 }
+	 if ($numeric =~ /\d/) {
+	    $ht{UTF_TO_NUMERIC}->{$utf8_source_string} = $numeric;
+	 }
+	 if ($annotation =~ /\S/) {
+	    $ht{UTF_ANNOTATION}->{$utf8_source_string} = $annotation;
+	 }
+         $n++;
+      }
+      close(IN);
+      # print STDERR "Loaded $n entries from $filename\n";
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub char_name_to_script {
+   local($this, $char_name, *ht) = @_;
+
+   return $cached_result if $cached_result = $ht{CHAR_NAME_TO_SCRIPT}->{$char_name};
+   $orig_char_name = $char_name;
+   $char_name =~ s/\s+(CONSONANT|LETTER|LIGATURE|SIGN|SYLLABLE|SYLLABICS|VOWEL)\b.*$//;
+   my $script_name;
+   while ($char_name) {
+      last if $script_name = $ht{SCRIPT_NORM}->{(uc $char_name)};
+      $char_name =~ s/\s*\S+\s*$//;
+   }
+   $script_name = "" unless defined($script_name);
+   $ht{CHAR_NAME_TO_SCRIPT}->{$char_name} = $script_name;
+   return $script_name;
+}
+
+sub letter_plus_char_p {
+   local($this, $char_name) = @_;
+
+   return $cached_result if $cached_result = $ht{CHAR_NAME_LETTER_PLUS}->{$char_name};
+   my $letter_plus_p = ($char_name =~ /\b(?:LETTER|VOWEL SIGN|AU LENGTH MARK|CONSONANT SIGN|SIGN VIRAMA|SIGN PAMAAEH|SIGN COENG|SIGN AL-LAKUNA|SIGN ASAT|SIGN ANUSVARA|SIGN ANUSVARAYA|SIGN BINDI|TIPPI|SIGN NIKAHIT|SIGN CANDRABINDU|SIGN VISARGA|SIGN REAHMUK|SIGN NUKTA|SIGN DOT BELOW|HEBREW POINT)\b/) ? 1 : 0;
+   $ht{CHAR_NAME_LETTER_PLUS}->{$char_name} = $letter_plus_p;
+   return $letter_plus_p;
+}
+
+sub subjoined_char_p {
+   local($this, $char_name) = @_;
+
+   return $cached_result if $cached_result = $ht{CHAR_NAME_SUBJOINED}->{$char_name};
+   my $subjoined_p = (($char_name =~ /\b(?:SUBJOINED LETTER|VOWEL SIGN|AU LENGTH MARK|EMPHASIS MARK|CONSONANT SIGN|SIGN VIRAMA|SIGN PAMAAEH|SIGN COENG|SIGN ASAT|SIGN ANUSVARA|SIGN ANUSVARAYA|SIGN BINDI|TIPPI|SIGN NIKAHIT|SIGN CANDRABINDU|SIGN VISARGA|SIGN REAHMUK|SIGN DOT BELOW|HEBREW (POINT|PUNCTUATION GERESH)|ARABIC (?:DAMMA|DAMMATAN|FATHA|FATHATAN|HAMZA|KASRA|KASRATAN|MADDAH|SHADDA|SUKUN))\b/)) ? 1 : 0;
+   $ht{CHAR_NAME_SUBJOINED}->{$char_name} = $subjoined_p;
+   return $subjoined_p;
+}
+
+sub new_node_id {
+   local($this, *chart_ht) = @_;
+
+   my $n_nodes = $chart_ht{N_NODES};
+   $n_nodes++;
+   $chart_ht{N_NODES} = $n_nodes;
+   return $n_nodes;
+}
+
+sub add_node {
+   local($this, $s, $start, $end, *chart_ht, $type, $comment) = @_;
+
+   my $node_id = $this->new_node_id(*chart_ht);
+   # print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if $comment =~ /number/;
+   # print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if ($start >= 0) && ($start < 50);
+   $chart_ht{NODE_START}->{$node_id} = $start;
+   $chart_ht{NODE_END}->{$node_id} = $end;
+   $chart_ht{NODES_STARTING_AT}->{$start}->{$node_id} = 1;
+   $chart_ht{NODES_ENDING_AT}->{$end}->{$node_id} = 1;
+   $chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}->{$node_id} = 1;
+   $chart_ht{NODE_TYPE}->{$node_id} = $type;
+   $chart_ht{NODE_COMMENT}->{$node_id} = $comment;
+   $chart_ht{NODE_ROMAN}->{$node_id} = $s;
+   return $node_id;
+}
+
+sub get_node_for_span {
+   local($this, $start, $end, *chart_ht) = @_;
+
+   return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+
+   return (@node_ids) ? $node_ids[0] : "";
+}
+
+sub get_node_for_span_and_type {
+   local($this, $start, $end, *chart_ht, $type) = @_;
+
+   return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+
+   foreach $node_id (@node_ids) {
+      return $node_id if $chart_ht{NODE_TYPE}->{$node_id} eq $type;
+   }
+   return "";
+}
+
+sub get_node_roman {
+   local($this, $node_id, *chart_id, $default) = @_;
+
+   $default = "" unless defined($default);
+   my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
+   return (defined($roman)) ? $roman : $default;
+}
+
+sub set_node_id_slot_value {
+   local($this, $node_id, $slot, $value, *chart_id) = @_;
+ 
+   $chart_ht{NODE_SLOT}->{$node_id}->{$slot} = $value;
+}
+
+sub copy_slot_values {
+   local($this, $old_node_id, $new_node_id, *chart_id, @slots) = @_;
+
+   if (@slots) {
+      foreach $slot (keys %{$chart_ht{NODE_SLOT}->{$old_node_id}}) {
+         if (($slots[0] eq "all") || $util->member($slot, @slots)) {
+	    my $value = $chart_ht{NODE_SLOT}->{$old_node_id}->{$slot};
+	    $chart_ht{NODE_SLOT}->{$new_node_id}->{$slot} = $value if defined($value);
+	 }
+      }
+   }
+}
+
+sub get_node_id_slot_value {
+   local($this, $node_id, $slot, *chart_id, $default) = @_;
+ 
+   $default = "" unless defined($default);
+   my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
+   return (defined($value)) ? $value : $default;
+}
+
+sub get_node_for_span_with_slot_value {
+   local($this, $start, $end, $slot, *chart_id, $default) = @_;
+
+   $default = "" unless defined($default);
+   return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+   foreach $node_id (@node_ids) {
+      my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
+      return $value if defined($value);
+   }
+   return $default;
+}
+
+sub get_node_for_span_with_slot {
+   local($this, $start, $end, $slot, *chart_id, $default) = @_;
+
+   $default = "" unless defined($default);
+   return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+   foreach $node_id (@node_ids) {
+      my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
+      return $node_id if defined($value);
+   }
+   return $default;
+}
+
+sub register_new_complex_number_span_segment {
+   local($this, $start, $mid, $end, *chart_id, $line_number) = @_;
+   # e.g. 4 10 (= 40); 20 5 (= 25)
+   # might become part of larger complex number span, e.g. 4 1000 3 100 20 1
+
+   # print STDERR "register_new_complex_number_span_segment $start-$mid-$end\n" if $line_number == 43;
+   if (defined($old_start = $chart_ht{COMPLEX_NUMERIC_END_START}->{$mid})) {
+      undef($chart_ht{COMPLEX_NUMERIC_END_START}->{$mid});
+      $chart_ht{COMPLEX_NUMERIC_START_END}->{$old_start} = $end;
+      $chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $old_start;
+   } else {
+      $chart_ht{COMPLEX_NUMERIC_START_END}->{$start} = $end;
+      $chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $start;
+   }
+}
+
+sub romanize_by_token_with_caching {
+   local($this, $s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number) = @_;
+
+   $control = "" unless defined($control);
+   my $return_chart_p = ($control =~ /return chart/i);
+   my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
+   return $this->romanize($s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number)
+     if $return_chart_p || $return_offset_mappings_p;
+   my $result = "";
+   my @separators = ();
+   my @tokens = ();
+   $s =~ s/\n$//; # Added May 2, 2019 as bug-fix (duplicate empty lines)
+   while (($sep, $token, $rest) = ($s =~ /^(\s*)(\S+)(.*)$/)) {
+      push(@separators, $sep);
+      push(@tokens, $token);
+      $s = $rest;
+   }
+   push(@separators, $s);
+   while (@tokens) {
+      my $sep = shift @separators;
+      my $token = shift @tokens;
+      $result .= $sep;
+      if ($token =~ /^[\x00-\x7F]*$/) { # all ASCII
+         $result .= $token;
+      } else {
+         my $rom_token = $ht{CACHED_ROMANIZATION}->{$lang_code}->{$token};
+         unless (defined($rom_token)) {
+            $rom_token = $this->romanize($token, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number);
+	    $ht{CACHED_ROMANIZATION}->{$lang_code}->{$token} = $rom_token if defined($rom_token);
+         }
+         $result .= $rom_token;
+      }
+   }
+   my $sep = shift @separators;
+   $result .= $sep if defined($sep);
+
+   return $result;
+}
+
+sub romanize {
+   local($this, $s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number, $initial_rom_char_offset) = @_;
+
+   my $orig_lang_code = $lang_code;
+   # Check whether the text (to be romanized) starts with a language code directive.
+   if (($line_lang_code) = ($s =~ /^::lcode\s+([a-z][a-z][a-z])\s/)) {
+      $lang_code = $line_lang_code;
+   }
+   $initial_char_offset = 0 unless defined($initial_char_offset);
+   $initial_rom_char_offset = 0 unless defined($initial_rom_char_offset);
+   $control = "" unless defined($control);
+   my $return_chart_p = ($control =~ /return chart/i);
+   my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
+   $line_number = "" unless defined($line_number);
+   my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   my $n_characters = $#chars + 1;
+   %chart_ht = ();
+   $chart_ht{N_CHARS} = $n_characters;
+   $chart_ht{N_NODES} = 0;
+   my $char = "";
+   my $char_name = "";
+   my $prev_script = "";
+   my $current_script = "";
+   my $script_start = 0;
+   my $script_end = 0;
+   my $prev_letter_plus_script = "";
+   my $current_letter_plus_script = "";
+   my $letter_plus_script_start = 0;
+   my $letter_plus_script_end = 0;
+   my $log ="";
+   my $n_right_to_left_chars = 0;
+   my $n_left_to_right_chars = 0;
+   my $hebrew_word_start = ""; # used to identify Hebrew words with points
+   my $hebrew_word_contains_point = 0;
+   my $current_word_start = "";
+   my $current_word_script = "";
+   my $braille_all_caps_p = 0;
+
+   # prep
+   foreach $i ((0 .. ($#chars + 1))) {
+      if ($i <= $#chars) {
+         $char = $chars[$i];
+         $chart_ht{ORIG_CHAR}->{$i} = $char;
+         $char_name = $ht{UTF_TO_CHAR_NAME}->{$char} || "";
+         $chart_ht{CHAR_NAME}->{$i} = $char_name;
+         $current_script = $this->char_name_to_script($char_name, *ht);
+	 $current_script_direction = $ht{DIRECTION}->{$current_script} || '';
+	 if ($current_script_direction eq 'right-to-left') {
+	    $n_right_to_left_chars++;
+	 } elsif (($char =~ /^[a-z]$/i) || ! ($char =~ /^[\x00-\x7F]$/)) {
+	    $n_left_to_right_chars++;
+	 }
+         $chart_ht{CHAR_SCRIPT}->{$i} = $current_script;
+         $chart_ht{SCRIPT_SEGMENT_START}->{$i} = ""; # default value, to be updated later
+         $chart_ht{SCRIPT_SEGMENT_END}->{$i} = "";   # default value, to be updated later
+         $chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = ""; # default value, to be updated later
+         $chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = "";   # default value, to be updated later
+	 $subjoined_char_p = $this->subjoined_char_p($char_name);
+	 $chart_ht{CHAR_SUBJOINED}->{$i} = $subjoined_char_p;
+	 $letter_plus_char_p = $this->letter_plus_char_p($char_name);
+	 $chart_ht{CHAR_LETTER_PLUS}->{$i} = $letter_plus_char_p;
+	 $current_letter_plus_script = ($letter_plus_char_p) ? $current_script : "";
+         $numeric_value = $ht{UTF_TO_NUMERIC}->{$char};
+         $numeric_value = "" unless defined($numeric_value);
+         $annotation = $ht{UTF_ANNOTATION}->{$char};
+         $annotation = "" unless defined($annotation);
+	 $chart_ht{CHAR_NUMERIC_VALUE}->{$i} = $numeric_value;
+	 $chart_ht{CHAR_ANNOTATION}->{$i} = $annotation;
+         $syllable_info = $ht{UTF_TO_SYLLABLE_INFO}->{$char} || "";
+	 $chart_ht{CHAR_SYLLABLE_INFO}->{$i} = $syllable_info;
+         $tone_mark = $ht{UTF_TO_TONE_MARK}->{$char} || "";
+	 $chart_ht{CHAR_TONE_MARK}->{$i} = $tone_mark;
+      } else {
+	 $char = "";
+         $char_name = "";
+	 $current_script = "";
+	 $current_letter_plus_script = "";
+      }
+      if ($char_name =~ /^HEBREW (LETTER|POINT|PUNCTUATION GERESH) /) {
+	 $hebrew_word_start = $i if $hebrew_word_start eq "";
+	 $hebrew_word_contains_point = 1 if $char_name =~ /^HEBREW POINT /;
+      } elsif ($hebrew_word_start ne "") {
+	 if ($hebrew_word_contains_point) {
+	    foreach $j (($hebrew_word_start .. ($i-1))) {
+	       $chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$j} = 1;
+	    }
+	    $chart_ht{CHAR_START_OF_WORD}->{$hebrew_word_start} = 1;
+	    $chart_ht{CHAR_END_OF_WORD}->{($i-1)} = 1;
+	 }
+	 $hebrew_word_start = "";
+	 $hebrew_word_contains_point = 0;
+      }
+      my $part_of_word_p = $current_script
+                        && ($this->letter_plus_char_p($char_name)
+                         || $this->subjoined_char_p($char_name)
+			 || ($char_name =~ /\b(LETTER|SYLLABLE|SYLLABICS|LIGATURE)\b/));
+
+      # Braille punctuation
+      my $end_offset = 0;
+      if ($char_name =~ /^Braille\b/i) {
+         if (($char =~ /^\s*$/) || ($char_name =~ /BLANK/)) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+	 } elsif ($chart_ht{NOT_PART_OF_WORD_P}->{$i}) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+	 } elsif ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$char}})
+	       || (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$char}})) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+	 } elsif (($i+1 <= $#chars) 
+	       && ($s1 = $char . $chars[$i+1])
+	       && ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s1}})
+	        || (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s1}}))) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+            $chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
+	 } elsif (($i+2 <= $#chars)
+	       && ($s2 = $char . $chars[$i+1] . $chars[$i+2])
+	       && ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s2}})
+	        || (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s2}}))) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+            $chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
+            $chart_ht{NOT_PART_OF_WORD_P}->{($i+2)} = 1;
+	 } elsif (($i+1 <= $#chars)
+	       && ($char eq $braille_capital_letter_indicator)
+	       && ($chars[$i+1] eq $braille_capital_letter_indicator)) {
+	    $braille_all_caps_p = 1;
+	 } else {
+	    $part_of_word_p = 1;
+	 }
+	 # last period in Braille text is also not part_of_word_p
+	 if (($char eq $braille_period)
+	  && (($i == $#chars)
+	   || (($i < $#chars)
+	    && (! $this->braille_string_p($chars[$i+1]))))) {
+	    $part_of_word_p = 0;
+	 }
+	 # period before other word-external punctuation is also not part_of_word_p
+	 if (($i > 0)
+	  && ($chars[$i-1] eq $braille_period)
+	  && (! $part_of_word_p)
+	  && ($current_word_start ne "")) {
+	    $end_offset = -1;
+	 }
+      } else {
+         $braille_all_caps_p = 0;
+      }
+      $chart_ht{BRAILLE_ALL_CAPS_P}->{$i} = $braille_all_caps_p;
+
+      if (($current_word_start ne "")
+       && ((! $part_of_word_p)
+        || ($current_script ne $current_word_script))) {
+         # END OF WORD
+	 $chart_ht{CHAR_START_OF_WORD}->{$current_word_start} = 1;
+	 $chart_ht{CHAR_END_OF_WORD}->{($i-1+$end_offset)} = 1;
+	 my $word = join("", @chars[$current_word_start .. ($i-1+$end_offset)]);
+	 $chart_ht{WORD_START_END}->{$current_word_start}->{$i} = $word;
+	 $chart_ht{WORD_END_START}->{$i+$end_offset}->{$current_word_start} = $word;
+	 # print STDERR "Word ($current_word_start-$i+$end_offset): $word ($current_word_script)\n";
+	 $current_word_start = "";
+	 $current_word_script = "";
+      }
+      if ($part_of_word_p && ($current_word_start eq "")) {
+         # START OF WORD
+	 $current_word_start = $i;
+	 $current_word_script = $current_script;
+      }
+      # print STDERR "$i char: $char ($current_script)\n";
+      unless ($current_script eq $prev_script) {
+	 if ($prev_script && ($i-1 >= $script_start)) {
+	    my $script_end = $i;
+            $chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start} = $script_end;
+            $chart_ht{SCRIPT_SEGMENT_END_TO_START}->{$script_end} = $script_start;
+	    foreach $i (($script_start .. $script_end)) {
+               $chart_ht{SCRIPT_SEGMENT_START}->{$i} = $script_start;
+               $chart_ht{SCRIPT_SEGMENT_END}->{$i} = $script_end;
+	    }
+	    # print STDERR "Script segment $script_start-$script_end: $prev_script\n";
+	 }
+	 $script_start = $i;
+      }
+      unless ($current_letter_plus_script eq $prev_letter_plus_script) {
+	 if ($prev_letter_plus_script && ($i-1 >= $letter_plus_script_start)) {
+	    my $letter_plus_script_end = $i;
+            $chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$letter_plus_script_start} = $letter_plus_script_end;
+            $chart_ht{LETTER_TOKEN_SEGMENT_END_TO_START}->{$letter_plus_script_end} = $letter_plus_script_start;
+	    foreach $i (($letter_plus_script_start .. $letter_plus_script_end)) {
+               $chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = $letter_plus_script_start;
+               $chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = $letter_plus_script_end;
+	    }
+	    # print STDERR "Script token segment $letter_plus_script_start-$letter_plus_script_end: $prev_letter_plus_script\n";
+	 }
+	 $letter_plus_script_start = $i;
+      }
+      $prev_script = $current_script;
+      $prev_letter_plus_script = $current_letter_plus_script;
+   }
+   $ht{STRING_IS_DOMINANTLY_RIGHT_TO_LEFT}->{$s} = 1 if $n_right_to_left_chars > $n_left_to_right_chars;
+
+   # main
+   my $i = 0;
+   while ($i <= $#chars) {
+      my $char = $chart_ht{ORIG_CHAR}->{$i};
+      my $current_script = $chart_ht{CHAR_SCRIPT}->{$i};
+      $chart_ht{CHART_CONTAINS_SCRIPT}->{$current_script} = 1;
+      my $script_segment_start = $chart_ht{SCRIPT_SEGMENT_START}->{$i};
+      my $script_segment_end = $chart_ht{SCRIPT_SEGMENT_END}->{$i};
+      my $char_name = $chart_ht{CHAR_NAME}->{$i};
+      my $subjoined_char_p = $chart_ht{CHAR_SUBJOINED}->{$i};
+      my $letter_plus_char_p = $chart_ht{CHAR_LETTER_PLUS}->{$i};
+      my $numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{$i};
+      my $annotation = $chart_ht{CHAR_ANNOTATION}->{$i};
+      # print STDERR "  $char_name annotation: $annotation\n" if $annotation;
+      my $tone_mark = $chart_ht{CHAR_TONE_MARK}->{$i};
+      my $found_char_mapping_p = 0;
+      my $prev_char_name = ($i >= 1) ? $chart_ht{CHAR_NAME}->{($i-1)} : "";
+      my $prev2_script = ($i >= 2) ? $chart_ht{CHAR_SCRIPT}->{($i-2)} : "";
+      my $prev_script = ($i >= 1) ? $chart_ht{CHAR_SCRIPT}->{($i-1)} : "";
+      my $next_script = ($i < $#chars) ? $chart_ht{CHAR_SCRIPT}->{($i+1)} : "";
+      my $next_char = ($i < $#chars) ? $chart_ht{ORIG_CHAR}->{($i+1)} : "";
+      my $next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char} || "";
+      my $prev2_letter_plus_char_p = ($i >= 2) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-2)} : 0;
+      my $prev_letter_plus_char_p = ($i >= 1) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-1)} : 0;
+      my $next_letter_plus_char_p = ($i < $#chars) ? $chart_ht{CHAR_LETTER_PLUS}->{($i+1)} : 0;
+      my $next_index = $i + 1;
+
+      # Braille numeric mode
+      if ($char eq $braille_number_indicator) {
+	 my $offset = 0;
+	 my $numeric_value = "";
+	 my $digit;
+         while ($i+$offset < $#chars) {
+	    $offset++;
+	    my $offset_char = $chart_ht{ORIG_CHAR}->{$i+$offset};
+            if (defined($digit = $ht{BRAILLE_TO_DIGIT}->{$offset_char})) {
+	       $numeric_value .= $digit;
+	    } elsif (($offset_char eq $braille_decimal_point)
+                  || ($ht{UTF_CHAR_MAPPING}->{$offset_char}->{"."})) {
+	       $numeric_value .= ".";
+	    } elsif ($offset_char eq $braille_comma) {
+	       $numeric_value .= ",";
+	    } elsif ($offset_char eq $braille_numeric_space) {
+	       $numeric_value .= " ";
+	    } elsif ($offset_char eq $braille_solidus) {
+	       $numeric_value .= "/";
+	    } elsif ($offset_char eq $braille_number_indicator) {
+	        # stay in Braille numeric mode
+	    } elsif ($offset_char eq $braille_letter_indicator) {
+	       # consider as part of number, but without contributing to numeric_value
+	       last;
+	    } else {
+	       $offset--; 
+	       last;
+	    }
+	 }
+	 if ($offset) {
+            $next_index = $i + $offset + 1;
+	    $node_id = $this->add_node($numeric_value, $i, $next_index, *chart_ht, "", "braille number");
+	    $found_char_mapping_p = 1;
+	 }
+      }
+
+      unless ($found_char_mapping_p) {
+        foreach $string_length (reverse(1 .. 6)) {
+	  next if ($i + $string_length-1) > $#chars;
+	  my $start_of_word_p = $chart_ht{CHAR_START_OF_WORD}->{$i} || 0;
+	  my $end_of_word_p = $chart_ht{CHAR_END_OF_WORD}->{($i+$string_length-1)} || 0;
+	  my $multi_char_substring = join("", @chars[$i..($i+$string_length-1)]);
+	  my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
+	  @mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
+	  my @mappings_whole = ();
+	  my @mappings_start_or_end = ();
+	  my @mappings_other = ();
+	  foreach $mapping (@mappings) {
+	    next if $mapping =~ /\(__.*__\)/;
+	    if ($ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	     || $ht{USE_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$mapping}) {
+	       push(@mappings_whole, $mapping) if $start_of_word_p && $end_of_word_p;
+	    } elsif ($ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	          || $ht{USE_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       push(@mappings_start_or_end, $mapping) if $start_of_word_p;
+	    } elsif ($ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	          || $ht{USE_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       push(@mappings_start_or_end, $mapping) if $end_of_word_p;
+	    } else {
+	       push(@mappings_other, $mapping);
+	    }
+	  }
+	  @mappings = @mappings_whole;
+	  @mappings = @mappings_start_or_end unless @mappings;
+	  @mappings = @mappings_other unless @mappings;
+	  foreach $mapping (@mappings) {
+	    next if $mapping =~ /\(__.*__\)/;
+	    if ($ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	     || $ht{DONT_USE_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       next if $start_of_word_p;
+	    }
+	    if ($ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	     || $ht{DONT_USE_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       next if $end_of_word_p;
+	    }
+	    my $mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $mapping) : $mapping;
+	    $node_id = $this->add_node($mapping2, $i, $i+$string_length, *chart_ht, "", "multi-char-mapping");
+	    $next_index = $i + $string_length;
+	    $found_char_mapping_p = 1;
+	    if ($annotation) {
+	       @annotation_elems = split(/,\s*/, $annotation);
+	       foreach $annotation_elem (@annotation_elems) {
+	          if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
+	             $this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
+		  } else {
+	             $this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
+		  }
+	       }
+	    }
+	  }
+	  my @alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
+	  @alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING}->{$multi_char_substring}} unless @alt_mappings;
+	  @alt_mappings = () if ($#alt_mappings == 0) && ($alt_mappings[0] eq "_NONE_");
+	  foreach $alt_mapping (@alt_mappings) {
+	    if ($chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$i}) {
+	       next unless
+	          $ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	       || $ht{USE_ALT_IN_POINTED}->{$multi_char_substring}->{$alt_mapping};
+	    }
+	    if ($ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	     || $ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$alt_mapping}) {
+	       next unless $start_of_word_p && $end_of_word_p;
+	    }
+	    if ($ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	     || $ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
+	       next unless $start_of_word_p;
+	    }
+	    if ($ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	     || $ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
+	       next unless $end_of_word_p;
+	    }
+	    my $alt_mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $alt_mapping) : $alt_mapping;
+	    $node_id = $this->add_node($alt_mapping2, $i, $i+$string_length, *chart_ht, "alt", "multi-char-mapping");
+	    if ($annotation) {
+	       @annotation_elems = split(/,\s*/, $annotation);
+	       foreach $annotation_elem (@annotation_elems) {
+	          if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
+	             $this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
+	          } else {
+	             $this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
+	          }
+	       }
+	    }
+	  }
+        }
+      }
+      unless ($found_char_mapping_p) {
+	 my $prev_node_id = $this->get_node_for_span($i-4, $i, *chart_ht)
+			 || $this->get_node_for_span($i-3, $i, *chart_ht)
+			 || $this->get_node_for_span($i-2, $i, *chart_ht)
+			 || $this->get_node_for_span($i-1, $i, *chart_ht);
+	 my $prev_char_roman = ($prev_node_id) ? $this->get_node_roman($prev_node_id, *chart_id) : "";
+	 my $prev_node_start = ($prev_node_id) ? $chart_ht{NODE_START}->{$prev_node_id} : "";
+
+	 # Number
+         if (($numeric_value =~ /\d/)
+	       && (! ($char_name =~ /SUPERSCRIPT/))) {
+	    my $prev_numeric_value = $this->get_node_for_span_with_slot_value($i-1, $i, "numeric-value", *chart_id);
+	    my $sep = "";
+	    $sep = " " if ($char_name =~ /^vulgar fraction /i) && ($prev_numeric_value =~ /\d/);
+	    $node_id = $this->add_node("$sep$numeric_value", $i, $i+1, *chart_ht, "", "number");
+	    $this->set_node_id_slot_value($node_id, "numeric-value", $numeric_value, *chart_ht);
+	    if ((($prev_numeric_value =~ /\d/) && ($numeric_value =~ /\d\d/))
+	     || (($prev_numeric_value =~ /\d\d/) && ($numeric_value =~ /\d/))) {
+	       # pull in any other parts of single digits
+	       my $j = 1;
+	       # pull in any single digits adjoining on left
+	       if ($prev_numeric_value =~ /^\d$/) {
+		  while (1) {
+	             if (($i-$j-1 >= 0)
+		      && defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-1, $i-$j, "numeric-value", *chart_id))
+		      && ($digit_value =~ /^\d$/)) {
+		        $j++;
+		     } elsif (($i-$j-2 >= 0)
+                           && ($chart_ht{ORIG_CHAR}->{($i-$j-1)} =~ /^[.,]$/)
+		           && defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-2, $i-$j-1, "numeric-value", *chart_id))
+		           && ($digit_value =~ /^\d$/)) {
+		        $j += 2;
+		     } else {
+		        last;
+		     }
+		  }
+	       }
+	       # pull in any single digits adjoining on right
+	       my $k = 0;
+	       if ($numeric_value =~ /^\d$/) {
+	          while (1) {
+	             if (defined($next_numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{($i+$k+1)})
+		      && ($next_numeric_value =~ /^\d$/)) {
+		        $k++;
+		     } else {
+		        last;
+		     }
+		  }
+	       }
+	       $this->register_new_complex_number_span_segment($i-$j, $i, $i+$k+1, *chart_ht, $line_number);
+	    }
+	    if ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
+	     && ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
+	       $de_accented_translit = $util->de_accent_string($tonal_translit);
+	       if ($numeric_value =~ /^(10000|1000000000000|10000000000000000)$/) {
+                  $chart_ht{NODE_TYPE}->{$node_id} = "alt"; # keep, but demote
+	          $alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
+	       } else {
+	          $alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "alt", "CJK");
+	       }
+            }
+
+	 # ASCII
+	 } elsif ($char =~ /^[\x00-\x7F]$/) {
+	    $this->add_node($char, $i, $i+1, *chart_ht, "", "ASCII"); # ASCII character, incl. control characters
+
+	 # Emoji, dingbats, pictographs
+	 } elsif ($char =~ /^(\xE2[\x98-\x9E]|\xF0\x9F[\x8C-\xA7])/) {
+	    $this->add_node($char, $i, $i+1, *chart_ht, "", "pictograph");
+
+         # Hangul (Korean)
+         } elsif (($char =~ /^[\xEA-\xED]/)
+	       && ($romanized_char = $this->unicode_hangul_romanization($char))) {
+	    $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "Hangul");
+
+         # CJK (Chinese, Japanese, Korean)
+	 } elsif ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
+	       && ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
+	    $de_accented_translit = $util->de_accent_string($tonal_translit);
+	    $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
+
+	 # Virama (cancel preceding vowel in Abudiga scripts)
+	 } elsif ($char_name =~ /\bSIGN (?:VIRAMA|AL-LAKUNA|ASAT|COENG|PAMAAEH)\b/) {
+	    # VIRAMA: cancel preceding default vowel (in Abudiga scripts)
+	    if (($prev_script eq $current_script)
+	     && (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
+	     && ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
+	       $this->add_node($prev_char_roman_consonant, $prev_node_start, $i+1, *chart_ht, "", "virama");
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-virama");
+	    }
+
+	 # Nukta (special (typically foreign) variant)
+	 } elsif ($char_name =~ /\bSIGN (?:NUKTA)\b/) {
+	    # NUKTA (dot): indicates special (typically foreign) variant; normally covered by multi-mappings
+	    if ($prev_script eq $current_script) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "nukta");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	       $this->set_node_id_slot_value($node_id, "nukta", 1, *chart_ht);
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-nukta");
+	    }
+
+	 # Zero-width character, incl. zero width space/non-joiner/joiner, left-to-right/right-to-left mark
+	 } elsif ($char =~ /^\xE2\x80[\x8B-\x8F\xAA-\xAE]$/) {
+	    if ($prev_node_id) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "zero-width-char");
+	    }
+	 } elsif (($char =~ /^\xEF\xBB\xBF$/) && $prev_node_id) { # OK to leave byte-order-mark at beginning of line
+	    my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
+            $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+
+	 # Tone mark
+	 } elsif ($tone_mark) {
+	    if ($prev_script eq $current_script) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "tone-mark");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	       $this->set_node_id_slot_value($node_id, "tone-mark", $tone_mark, *chart_ht);
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-tone-mark");
+	    }
+
+	 # Diacritic
+	 } elsif (($char_name =~ /\b(ACCENT|TONE|COMBINING DIAERESIS|COMBINING DIAERESIS BELOW|COMBINING MACRON|COMBINING VERTICAL LINE ABOVE|COMBINING DOT ABOVE RIGHT|COMBINING TILDE|COMBINING CYRILLIC|MUUSIKATOAN|TRIISAP)\b/) && ($ht{UTF_TO_CAT}->{$char} =~ /^Mn/)) {
+	    if ($prev_script eq $current_script) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "diacritic");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	       $diacritic = lc $char_name;
+	       $diacritic =~ s/^.*(?:COMBINING CYRILLIC|COMBINING|SIGN)\s+//i;
+	       $diacritic =~ s/^.*(ACCENT|TONE)/$1/i;
+	       $diacritic =~ s/^\s*//;
+	       $this->set_node_id_slot_value($node_id, "diacritic", $diacritic, *chart_ht);
+	       # print STDERR "diacritic: $diacritic\n";
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-diacritic");
+	    }
+
+	 # Romanize to find out more
+	 } elsif ($char_name) {
+	    if (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht))) {
+	       # print STDERR "ROM l.$line_number/$i: $romanized_char\n" if $line_number =~ /^[12]$/;
+	       print STDOUT "ROM l.$line_number/$i: $romanized_char\n" if $verbosePM;
+
+	       # Empty string mapping
+	       if ($romanized_char eq "\"\"") {
+	          $this->add_node("", $i, $i+1, *chart_ht, "", "empty-string-mapping");
+               # consider adding something for implausible romanizations of length 6+
+
+	       # keep original character (instead of romanized_char lengthener, character-18b00 etc.)
+	       } elsif (($romanized_char =~ /^(character|lengthener|modifier)/)) {
+	          $this->add_node($char, $i, $i+1, *chart_ht, "", "nevermind-keep-original");
+
+	       # Syllabic suffix in Abudiga languages, e.g. -m, -ng
+               } elsif (($romanized_char =~ /^\+(H|M|N|NG)$/i)
+		     && ($prev_script eq $current_script)
+		     && ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{"a"})) {
+		  my $core_suffix = $romanized_char;
+		  $core_suffix =~ s/^\+//;
+		  if ($prev_char_roman =~ /[aeiou]$/i) {
+	             $this->add_node($core_suffix, $i, $i+1, *chart_ht, "", "syllable-end-consonant");
+		  } else {
+	             $this->add_node(join("", $prev_char_roman, "a", $core_suffix), $prev_node_start, $i+1, *chart_ht, "", "syllable-end-consonant-with-added-a");
+	             $this->add_node(join("", "a", $core_suffix), $i, $i+1, *chart_ht, "backup", "syllable-end-consonant");
+		  }
+
+	       # Japanese special cases
+	       } elsif ($char_name =~ /(?:HIRAGANA|KATAKANA) LETTER SMALL Y/) {
+		  if (($prev_script eq $current_script)
+		   && (($prev_char_roman_consonant) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])i$/i))) {
+                     unless ($this->get_node_for_span_and_type($prev_node_start, $i+1, *chart_ht, "")) {
+	                $this->add_node("$prev_char_roman_consonant$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "japanese-contraction");
+		     }
+		  } else {
+	             $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "unexpected-japanese-contraction-character");
+		  }
+	       } elsif (($prev_script =~ /^(HIRAGANA|KATAKANA)$/i)
+		     && ($char_name eq "KATAKANA-HIRAGANA PROLONGED SOUND MARK") # Choonpu
+		     && (($prev_char_roman_vowel) = ($prev_char_roman =~ /([aeiou])$/i))) {
+	          $this->add_node("$prev_char_roman$prev_char_roman_vowel", $prev_node_start, $i+1, *chart_ht, "", "japanese-vowel-lengthening");
+	       } elsif (($current_script =~ /^(Hiragana|Katakana)$/i)
+	             && ($char_name =~ /^(HIRAGANA|KATAKANA) LETTER SMALL TU$/i) # Sokuon/Sukun
+		     && ($next_script eq $current_script)
+	             && ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, *ht, *chart_ht))
+		     && (($doubled_consonant) = ($romanized_next_char =~ /^(ch|[bcdfghjklmnpqrstwz])/i))) {
+		  # Note: $romanized_next_char could be part of a multi-character mapping
+		  # print STDERR "current_script: $current_script char_name: $char_name next_script: $next_script romanized_next_char: $romanized_next_char doubled_consonant: $doubled_consonant\n";
+		  $doubled_consonant = "t" if $doubled_consonant eq "ch";
+	          $this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "japanese-consonant-doubling");
+ 
+               # Greek small letter mu to micro-sign (instead of to "m") as used in abbreviations for microgram/micrometer/microliter/microsecond/micromolar/microfarad etc.
+               } elsif (($char_name eq "GREEK SMALL LETTER MU")
+	             && (! ($prev_script =~ /^GREEK$/))
+		     && ($i < $#chars)
+		     && ($chart_ht{ORIG_CHAR}->{($i+1)} =~ /^[cfgjlmstv]$/i)) {
+	          $this->add_node("\xC2\xB5", $i, $i+1, *chart_ht, "", "greek-mu-to-micro-sign");
+
+	       # Gurmukhi addak (doubles following consonant)
+               } elsif (($current_script eq "Gurmukhi")
+		     && ($char_name eq "GURMUKHI ADDAK")) {
+                  if (($next_script eq $current_script)
+		   && ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, *ht, *chart_ht))
+	           && (($doubled_consonant) = ($romanized_next_char =~ /^([bcdfghjklmnpqrstvwxz])/i))) {
+	             $this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "gurmukhi-consonant-doubling");
+		  } else {
+	             $this->add_node("'", $i, $i+1, *chart_ht, "", "gurmukhi-unexpected-addak");
+		  }
+
+	       # Subjoined character
+               } elsif ($subjoined_char_p
+		     && ($prev_script eq $current_script)
+	             && (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
+	             && ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
+		  my $new_roman = "$prev_char_roman_consonant$romanized_char";
+	          $this->add_node($new_roman, $prev_node_start, $i+1, *chart_ht, "", "subjoined-character");
+	          # print STDERR "  Subjoin l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;
+
+	       # Thai special case: written-pre-consonant-spoken-post-consonant
+	       } elsif (($char_name =~ /THAI CHARACTER/)
+		     && ($prev_script eq $current_script)
+		     && ($chart_ht{CHAR_SYLLABLE_INFO}->{($i-1)} =~ /written-pre-consonant-spoken-post-consonant/i)
+		     && ($prev_char_roman =~ /^[aeiou]+$/i)
+		     && ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]/)) {
+	          $this->add_node("$romanized_char$prev_char_roman", $prev_node_start, $i+1, *chart_ht, "", "thai-vowel-consonant-swap");
+
+	       # Thai special case: THAI CHARACTER O ANG (U+0E2D "\xE0\xB8\xAD")
+	       } elsif ($char_name eq "THAI CHARACTER O ANG") {
+		  if ($prev_script ne $current_script) {
+	             $this->add_node("", $i, $i+1, *chart_ht, "", "thai-initial-o-ang-drop");
+		  } elsif ($next_script ne $current_script) {
+	             $this->add_node("", $i, $i+1, *chart_ht, "", "thai-final-o-ang-drop");
+		  } else {
+	             my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
+		     my $romanized_prev2_char = $this->romanize_char_at_position($i-2, $lang_code, $output_style, *ht, *chart_ht);
+		     if (($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
+		      && ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
+	                $this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonants
+		     } elsif (($prev2_script eq $current_script)
+			   && 0
+		           && ($prev_char_name =~ /^THAI CHARACTER MAI [A-Z]+$/) # Thai tone
+			   && ($romanized_prev2_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
+			   && ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
+	                $this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonant+tone-mark and consonant
+		     } else {
+	                $this->add_node("", $i, $i+1, *chart_ht, "", "thai-middle-o-ang-drop"); # drop next to vowel
+		     }
+		  }
+
+	       # Romanization with space
+	       } elsif ($romanized_char =~ /\s/) {
+	          $this->add_node($char, $i, $i+1, *chart_ht, "", "space");
+
+	       # Tibetan special cases
+	       } elsif ($current_script eq "Tibetan") {
+
+                  if ($subjoined_char_p
+		   && ($prev_script eq $current_script)
+		   && $prev_letter_plus_char_p
+	           && ($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
+	             $this->add_node("$prev_char_roman$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "subjoined-tibetan-character");
+		  } elsif ($romanized_char =~ /^-A$/i) {
+	             my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
+		     if (! $prev_letter_plus_char_p) {
+	                $this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-frontal-dash-a");
+		     } elsif (($prev_script eq $current_script)
+		           && ($next_script eq $current_script)
+			   && ($prev_char_roman =~ /[bcdfghjklmnpqrstvwxyz]$/)
+			   && ($romanized_next_char =~ /^[aeiou]/)) {
+			$this->add_node("a'", $i, $i+1, *chart_ht, "", "tibetan-medial-dash-a");
+		     } elsif (($prev_script eq $current_script)
+		           && ($next_script eq $current_script)
+			   && ($prev_char_roman =~ /[aeiou]$/)
+			   && ($romanized_next_char =~ /[aeiou]/)) {
+			$this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-reduced-medial-dash-a");
+		     } elsif (($prev_script eq $current_script)
+		           && (! ($prev_char_roman =~ /[aeiou]/))
+			   && (! $next_letter_plus_char_p)) {
+			$this->add_node("a", $i, $i+1, *chart_ht, "", "tibetan-final-dash-a");
+		     } else {
+			$this->add_node("a", $i, $i+1, *chart_ht, "", "unexpected-tibetan-dash-a");
+		     }
+		  } elsif (($romanized_char =~ /^[AEIOU]/i)
+			&& ($prev_script eq $current_script)
+		        && ($prev_char_roman =~ /^A$/i)
+			&& (! $prev2_letter_plus_char_p)) {
+	             $this->add_node($romanized_char, $prev_node_start, $i+1, *chart_ht, "", "tibetan-dropped-word-initial-a");
+		  } else {
+	             $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
+		  }
+
+               # Khmer (for MUUSIKATOAN etc. see under "Diacritic" above)
+	       } elsif (($current_script eq "Khmer")
+	             && (($char_roman_consonant, $char_roman_vowel) = ($romanized_char =~ /^(.*[bcdfghjklmnpqrstvwxyz])([ao]+)-$/i))) {
+	           my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
+		   if (($next_script eq $current_script)
+		    && ($romanized_next_char =~ /^[aeiouy]/i)) {
+                      $this->add_node($char_roman_consonant, $i, $i+1, *chart_ht, "", "khmer-vowel-drop");
+		   } else {
+                      $this->add_node("$char_roman_consonant$char_roman_vowel", $i, $i+1, *chart_ht, "", "khmer-standard-unicode-based-romanization");
+		   }
+
+	       # Abudiga add default vowel
+	       } elsif ((@abudiga_default_vowels = sort keys %{$ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}})
+		     && ($abudiga_default_vowel = $abudiga_default_vowels[0])
+	             && ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
+		  my $new_roman = join("", $romanized_char, $abudiga_default_vowel);
+	          $this->add_node($new_roman, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization-plus-abudiga-default-vowel");
+	          # print STDERR "  Abudiga add default vowel l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;
+
+	       # Standard romanization
+	       } else {
+	          $node_id = $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
+	       }
+	    } else {
+	       $this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original");
+	    }
+	 } elsif (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht))
+	       && ((length($romanized_char) <= 2)
+	        || ($ht{UTF_TO_CHAR_ROMANIZATION}->{$char}))) { # or from unicode_overwrite_romanization table
+	    $romanized_char =~ s/^""$//;
+	    $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "romanized-without-character-name");
+	 } else {
+	    $this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original-without-character-name");
+	 }
+      }
+      $i = $next_index;
+   }
+
+   $this->schwa_deletion(0, $n_characters, *chart_ht, $lang_code);
+   $this->default_vowelize_tibetan(0, $n_characters, *chart_ht, $lang_code, $line_number) if $chart_ht{CHART_CONTAINS_SCRIPT}->{"Tibetan"};
+   $this->assemble_numbers_in_chart(*chart_ht, $line_number);
+
+   if ($return_chart_p) {
+   } elsif ($return_offset_mappings_p) {
+      ($result, $offset_mappings, $new_char_offset, $new_rom_char_offset) = $this->best_romanized_string(0, $n_characters, *chart_ht, $control, $initial_char_offset, $initial_rom_char_offset);
+   } else {
+      $result = $this->best_romanized_string(0, $n_characters, *chart_ht) unless $return_chart_p;
+   }
+
+   if ($verbosePM) {
+      my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-log.txt";
+      $util->append_to_file($logfile, $log) if $log && (-r $logfile);
+   }
+
+   return ($result, $offset_mappings) if $return_offset_mappings_p;
+   return *chart_ht if $return_chart_p;
+   return $result;
+}
+
+sub string_to_json_string {
+   local($this, $s) = @_;
+
+   utf8::decode($s);
+   my $j = JSON->new->utf8->encode([$s]);
+   $j =~ s/^\[(.*)\]$/$1/;
+   return $j;
+}
+
+sub chart_to_json_romanization_elements {
+   local($this, $chart_start, $chart_end, *chart_ht, $line_number) = @_;
+
+   my $result = "";
+   my $start = $chart_start;
+   my $end;
+   while ($start < $chart_end) {
+      $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my @best_romanizations;
+      if (($end && ($start < $end))
+       && (@best_romanizations = $this->best_romanizations($start, $end, *chart_ht))) {
+         $orig_segment = $this->orig_string_at_span($start, $end, *chart_ht);
+         $next_start = $end;
+      } else {
+         $orig_segment = $chart_ht{ORIG_CHAR}->{$start};
+         @best_romanizations = ($orig);
+	 $next_start = $start + 1;
+      }
+      $exclusive_end = $end - 1;
+      # $guarded_orig = $util->string_guard($orig_segment);
+      $guarded_orig = $this->string_to_json_string($orig_segment);
+      $result .= "  { \"line\": $line_number, \"start\": $start, \"end\": $exclusive_end, \"orig\": $guarded_orig, \"roms\": [";
+      foreach $i ((0 .. $#best_romanizations)) {
+         my $rom = $best_romanizations[$i];
+	 # my $guarded_rom = $util->string_guard($rom);
+	 my $guarded_rom = $this->string_to_json_string($rom);
+	 $result .= " { \"rom\": $guarded_rom";
+	 # $result .= ", \"alt\": true" if $i >= 1;
+	 $result .= " }";
+	 $result .= "," if $i < $#best_romanizations;
+      }
+      $result .= " ] },\n";
+      $start = $next_start;
+   }
+   return $result;
+}
+
+sub default_vowelize_tibetan {
+   local($this, $chart_start, $chart_end, *chart_ht, $lang_code, $line_number) = @_;
+
+   # my $verbose = ($line_number == 103);
+   # print STDERR "\nStart default_vowelize_tibetan l.$line_number $chart_start-$chart_end\n" if $verbose;
+   my $token_start = $chart_start;
+   my $next_token_start = $chart_start;
+   while (($token_start = $next_token_start) < $chart_end) {
+      $next_token_start = $token_start + 1;
+
+      next unless $chart_ht{CHAR_LETTER_PLUS}->{$token_start};
+      my $current_script = $chart_ht{CHAR_SCRIPT}->{$token_start};
+         next unless ($current_script eq "Tibetan");
+      my $token_end = $chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$token_start};
+	 next unless $token_end;
+	 next unless $token_end > $token_start;
+      $next_token_start = $token_end;
+
+      my $start = $token_start;
+      my $end;
+      my @node_ids = ();
+      while ($start < $token_end) {
+         $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+	 last unless $end && ($end > $start);
+         my @alt_node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+         last unless @alt_node_ids;
+         push(@node_ids, $alt_node_ids[0]);
+	 $start = $end;
+      }
+      my $contains_vowel_p = 0;
+      my @romanizations = ();
+      foreach $node_id (@node_ids) {
+         my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
+	 $roman = "" unless defined($roman);
+	 push(@romanizations, $roman);
+	 $contains_vowel_p = 1 if $roman =~ /[aeiou]/i;
+      }
+      # print STDERR "   old: $token_start-$token_end @romanizations\n" if $verbose;
+      unless ($contains_vowel_p) {
+	 my $default_vowel_target_index;
+	 if ($#node_ids <= 1) {
+	    $default_vowel_target_index = 0;
+	 } elsif ($romanizations[$#romanizations] eq "s") {
+	    if ($romanizations[($#romanizations-1)] eq "y") {
+	       $default_vowel_target_index = $#romanizations-1;
+	    } else {
+	       $default_vowel_target_index = $#romanizations-2;
+	    }
+	 } else {
+	    $default_vowel_target_index = $#romanizations-1;
+	 }
+	 $romanizations[$default_vowel_target_index] .= "a";
+	 my $old_node_id = $node_ids[$default_vowel_target_index];
+         my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+         my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
+	 my $new_roman = $old_roman . "a";
+	 my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-default-vowel");
+         $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+         $chart_ht{NODE_TYPE}->{$old_node_id} = "backup"; # keep, but demote
+      }
+      if (($romanizations[0] eq "'")
+       && ($#romanizations >= 1)
+       && ($romanizations[1] =~ /^[o]$/)) {
+	 my $old_node_id = $node_ids[0];
+         my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+         my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $new_node_id = $this->add_node("", $old_start, $old_end, *chart_ht, "", "tibetan-delete-apostrophe");
+	 $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+	 $chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
+      }
+      if (($#node_ids >= 1)
+       && ($romanizations[$#romanizations] =~ /^[bcdfghjklmnpqrstvwxz]+y$/)) {
+	 my $old_node_id = $node_ids[$#romanizations];
+         my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+         my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
+	 my $new_roman = $old_roman . "a";
+	 my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-final-vowel");
+	 $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+	 $chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
+      }
+      foreach $old_node_id (@node_ids) {
+         my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
+	 next unless $old_roman =~ /-a/;
+	 my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+	 my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $new_roman = $old_roman;
+	 $new_roman =~ s/-a/a/;
+	 my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-delete-dash");
+	 $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+	 $chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
+      }
+   }
+}
+
+sub schwa_deletion {
+   local($this, $chart_start, $chart_end, *chart_ht, $lang_code) = @_;
+   # delete word-final simple "a" in Devanagari (e.g. nepaala -> nepaal)
+   # see Wikipedia article "Schwa deletion in Indo-Aryan languages"
+
+   if ($chart_ht{CHART_CONTAINS_SCRIPT}->{"Devanagari"}) {
+      my $script_start = $chart_start;
+      my $next_script_start = $chart_start;
+      while (($script_start = $next_script_start) < $chart_end) {
+         $next_script_start = $script_start + 1;
+
+         my $current_script = $chart_ht{CHAR_SCRIPT}->{$script_start};
+	    next unless ($current_script eq "Devanagari");
+	 my $script_end = $chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start};
+	    next unless $script_end;
+	    next unless $script_end - $script_start >= 2;
+	 $next_script_start = $script_end;
+	 my $end_node_id = $this->get_node_for_span($script_end-1, $script_end, *chart_ht);
+	    next unless $end_node_id;
+         my $end_roman = $chart_ht{NODE_ROMAN}->{$end_node_id};
+	 next unless ($end_consonant) = ($end_roman =~ /^([bcdfghjklmnpqrstvwxz]+)a$/i);
+	 my $prev_node_id = $this->get_node_for_span($script_end-4, $script_end-1, *chart_ht)
+	                 || $this->get_node_for_span($script_end-3, $script_end-1, *chart_ht)
+	                 || $this->get_node_for_span($script_end-2, $script_end-1, *chart_ht);
+	    next unless $prev_node_id;
+         my $prev_roman = $chart_ht{NODE_ROMAN}->{$prev_node_id};
+	 next unless $prev_roman =~ /[aeiou]/i;
+	 # TO DO: check further back for vowel (e.g. if $prev_roman eq "r" due to vowel cancelation)
+	 
+         $chart_ht{NODE_TYPE}->{$end_node_id} = "alt"; # keep, but demote
+	 # print STDERR "* Schwa deletion " . ($script_end-1) . "-$script_end $end_roman->$end_consonant\n";
+	 $this->add_node($end_consonant, $script_end-1, $script_end, *chart_ht, "", "devanagari-with-deleted-final-schwa");
+      }
+   }
+}
+
+sub best_romanized_string {
+   local($this, $chart_start, $chart_end, *chart_ht, $control, $orig_char_offset, $rom_char_offset) = @_;
+
+   $control = "" unless defined($control);
+   my $current_orig_char_offset = $orig_char_offset || 0;
+   my $current_rom_char_offset = $rom_char_offset || 0;
+   my $return_offset_mappings_p = ($control =~ /\breturn offset mappings\b/);
+   my $result = "";
+   my $start = $chart_start;
+   my $end;
+   my @char_offsets = ("$current_orig_char_offset:$current_rom_char_offset");
+   while ($start < $chart_end) {
+      $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my $n_orig_chars_in_segment = 0;
+      my $n_rom_chars_in_segment = 0;
+      if ($end && ($start < $end)) {
+         my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
+	 my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
+	 if (defined($best_romanization)) {
+            $result .= $best_romanization;
+	    if ($return_offset_mappings_p) {
+	       $n_orig_chars_in_segment = $end-$start;
+	       $n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
+	    }
+	    $start = $end;
+	 } else {
+	    my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
+            $result .= $best_romanization;
+	    $start++;
+	    if ($return_offset_mappings_p) {
+	       $n_orig_chars_in_segment = 1;
+	       $n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
+	    }
+	 }
+      } else {
+	 my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
+         $result .= $best_romanization;
+	 $start++;
+	 if ($return_offset_mappings_p) {
+	    $n_orig_chars_in_segment = 1;
+	    $n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
+	 }
+      }
+      if ($return_offset_mappings_p) {
+         my $new_orig_char_offset = $current_orig_char_offset + $n_orig_chars_in_segment;
+         my $new_rom_char_offset = $current_rom_char_offset + $n_rom_chars_in_segment;
+         my $offset_mapping = "$new_orig_char_offset:$new_rom_char_offset";
+         push(@char_offsets, $offset_mapping);
+         $current_orig_char_offset = $new_orig_char_offset;
+         $current_rom_char_offset = $new_rom_char_offset;
+      }
+   }
+   return ($result, join(",", @char_offsets), $current_orig_char_offset, $current_rom_char_offset) if $return_offset_mappings_p;
+   return $result;
+}
+
+sub orig_string_at_span {
+   local($this, $start, $end, *chart_ht) = @_;
+
+   my $result = "";
+   foreach $i (($start .. ($end-1))) {
+      $result .= $chart_ht{ORIG_CHAR}->{$i};
+   }
+   return $result;
+}
+
+sub find_end_of_rom_segment {
+   local($this, $start, $chart_end, *chart_ht) = @_;
+
+   my @ends = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}};
+   my $end_index = $#ends;
+   while (($end_index >= 0) && ($ends[$end_index] > $chart_end)) {
+      $end_index--;
+   }
+   if (($end_index >= 0)
+    && defined($end = $ends[$end_index])
+    && ($start < $end)) {
+      return $end;
+   } else {
+      return "";
+   }
+}
+
+sub best_romanizations {
+   local($this, $start, $end, *chart_ht) = @_;
+
+   @regular_romanizations = ();
+   @alt_romanizations = ();
+   @backup_romanizations = ();
+
+   foreach $node_id (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}}) {
+      my $type = $chart_ht{NODE_TYPE}->{$node_id};
+      my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
+      if (! defined($roman)) {
+         # ignore
+      } elsif (($type eq "backup") && ! defined($backup_romanization)) {
+         push(@backup_romanizations, $roman) unless $util->member($roman, @backup_romanizations);
+      } elsif (($type eq "alt") && ! defined($alt_romanization)) {
+         push(@alt_romanizations, $roman) unless $util->member($roman, @alt_romanizations);
+      } else {
+         push(@regular_romanizations, $roman) unless $util->member($roman, @regular_romanizations);
+      }
+   }
+   @regular_alt_romanizations = sort @regular_romanizations;
+   foreach $alt_romanization (sort @alt_romanizations) {
+      push(@regular_alt_romanizations, $alt_romanization) unless $util->member($alt_romanization, @regular_alt_romanizations);
+   }
+   return @regular_alt_romanizations if @regular_alt_romanizations;
+   return sort @backup_romanizations;
+}
+
+sub join_alt_romanizations_for_viz {
+   local($this, @list) = @_;
+
+   my @viz_romanizations = ();
+
+   foreach $alt_rom (@list) {
+      if ($alt_rom eq "") {
+         push(@viz_romanizations, "-");
+      } else {
+         push(@viz_romanizations, $alt_rom);
+      }
+   }
+   return join(", ", @viz_romanizations);
+}
+
+sub markup_orig_rom_strings {
+   local($this, $chart_start, $chart_end, *ht, *chart_ht, *pinyin_ht, $last_group_id_index) = @_;
+
+   my $marked_up_rom = "";
+   my $marked_up_orig = "";
+   my $start = $chart_start;
+   my $end;
+   while ($start < $chart_end) {
+      my $segment_start = $start;
+      my $segment_end = $start+1;
+      my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my $rom_segment = "";
+      my $orig_segment = "";
+      my $rom_title = "";
+      my $orig_title = "";
+      my $contains_alt_romanizations = 0;
+      if ($end) {
+	 $segment_end = $end;
+         my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
+	 my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
+	 if (defined($best_romanization)) {
+            $rom_segment .= $best_romanization;
+            $orig_segment .= $this->orig_string_at_span($start, $end, *chart_ht);
+	    $segment_end = $end;
+	    if ($#best_romanizations >= 1) {
+	       $rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
+	       $contains_alt_romanizations = 1;
+	    }
+	 } else {
+	    my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
+            $rom_segment .= $segment;
+            $orig_segment .= $segment;
+	    $segment_end = $start+1;
+	 }
+	 $start = $segment_end;
+      } else {
+         $rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
+         $orig_segment .= $this->orig_string_at_span($start, $start+1, *chart_ht);
+	 $segment_end = $start+1;
+	 $start = $segment_end;
+      }
+      my $next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
+      my $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+      while ($next_char_is_combining_p
+          && ($segment_end < $chart_end)
+	  && ($end = $this->find_end_of_rom_segment($segment_end, $chart_end, *chart_ht))
+	  && ($end > $segment_end)
+	  && (@best_romanizations = $this->best_romanizations($segment_end, $end, *chart_ht))
+	  && defined($best_romanization = $best_romanizations[0])) {
+         $orig_segment .= $this->orig_string_at_span($segment_end, $end, *chart_ht);
+	 $rom_segment .= $best_romanization;
+	 if ($#best_romanizations >= 1) {
+	    $rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
+	    $contains_alt_romanizations = 1;
+	 }
+	 $segment_end = $end;
+	 $start = $segment_end;
+	 $next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
+	 $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+      }
+      foreach $i (($segment_start .. ($segment_end-1))) {
+	 $orig_title .= "+&#x200E; &#x200E;" unless $orig_title eq "";
+         my $char = $chart_ht{ORIG_CHAR}->{$i};
+	 my $numeric = $ht{UTF_TO_NUMERIC}->{$char};
+	 $numeric = "" unless defined($numeric);
+	 my $pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
+	 $pic_descr = "" unless defined($pic_descr);
+	 if ($char =~ /^\xE4\xB7[\x80-\xBF]$/) {
+	    $orig_title .= "$char_name\n";
+	 } elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
+	    my $unicode = $utf8->utf8_to_unicode($char);
+	    $orig_title .= "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode)) . "\n";
+	    $orig_title .= "Chinese: $tonal_translit\n" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
+	    $orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
+	 } elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
+	    $orig_title .= "$char_name\n";
+	    $orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
+	    $orig_title .= "Picture: $pic_descr\n" if $pic_descr =~ /\S/;
+	 } else {
+	    my $unicode = $utf8->utf8_to_unicode($char);
+	    if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
+	       $orig_title .= "Hangul syllable U+" . (uc sprintf("%04x", $unicode)) . "\n";
+	    } else {
+	       $orig_title .= "Unicode character U+" . (uc sprintf("%04x", $unicode)) . "\n";
+	    }
+	 }
+      }
+      (@non_ascii_roms) = ($rom_segment =~ /([\xC0-\xFF][\x80-\xBF]*)/g);
+      foreach $char (@non_ascii_roms) {
+	 my $char_name = $ht{UTF_TO_CHAR_NAME}->{$char};
+         my $unicode = $utf8->utf8_to_unicode($char);
+	 my $unicode_s = "U+" . (uc sprintf("%04x", $unicode));
+	 if ($char_name) {
+	    $rom_title .= "$char_name\n";
+	 } else {
+	    $rom_title .= "$unicode_s\n";
+	 }
+      }
+      $last_group_id_index++;
+      $rom_title =~ s/\s*$//;
+      $rom_title =~ s/\n/&#xA;/g;
+      $orig_title =~ s/\s*$//;
+      $orig_title =~ s/\n/&#xA;&#x200E;/g;
+      $orig_title = "&#x202D;" . $orig_title . "&#x202C;";
+      my $rom_title_clause  = ($rom_title  eq "") ? "" : " title=\"$rom_title\"";
+      my $orig_title_clause = ($orig_title eq "") ? "" : " title=\"$orig_title\"";
+      my $alt_rom_clause = ($contains_alt_romanizations) ? "border-bottom:1px dotted;" : "";
+      $marked_up_rom .= "<span id=\"span-$last_group_id_index-1\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\" style=\"color:#00BB00;$alt_rom_clause\"$rom_title_clause>" . $util->guard_html($rom_segment) . "<\/span>";
+      $marked_up_orig .= "<span id=\"span-$last_group_id_index-2\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\"$orig_title_clause>" . $util->guard_html($orig_segment) . "<\/span>";
+      if (($last_char = $chart_ht{ORIG_CHAR}->{($segment_end-1)})
+       && ($last_char_name = $ht{UTF_TO_CHAR_NAME}->{$last_char})
+       && ($last_char_name =~ /^(FULLWIDTH COLON|FULLWIDTH COMMA|FULLWIDTH RIGHT PARENTHESIS|IDEOGRAPHIC COMMA|IDEOGRAPHIC FULL STOP|RIGHT CORNER BRACKET|BRAILLE PATTERN BLANK|TIBETAN MARK .*)$/)) {
+         $marked_up_orig .= "<wbr>";
+         $marked_up_rom .= "<wbr>";
+      }
+   }
+   return ($marked_up_rom, $marked_up_orig, $last_group_id_index);
+}
+
+sub romanizations_with_alternatives {
+   local($this, *ht, *chart_ht, *pinyin_ht, $chart_start, $chart_end) = @_;
+
+   $chart_start = 0 unless defined($chart_start);
+   $chart_end = $chart_ht{N_CHARS} unless defined($chart_end);
+   my $result = "";
+   my $start = $chart_start;
+   my $end;
+   # print STDOUT "romanizations_with_alternatives $chart_start-$chart_end\n";
+   while ($start < $chart_end) {
+      my $segment_start = $start;
+      my $segment_end = $start+1;
+      my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my $rom_segment = "";
+      # print STDOUT "  $start-$end\n";
+      if ($end) {
+	 $segment_end = $end;
+         my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
+         # print STDOUT "  $start-$end @best_romanizations\n";
+	 if (@best_romanizations) {
+	    if ($#best_romanizations == 0) {
+	       $rom_segment .= $best_romanizations[0];
+	    } else {
+	       $rom_segment .= "{" . join("|", @best_romanizations) . "}";
+	    }
+	    $segment_end = $end;
+	 } else {
+	    my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
+            $rom_segment .= $segment;
+	    $segment_end = $start+1;
+	 }
+	 $start = $segment_end;
+      } else {
+         $rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
+	 $segment_end = $start+1;
+	 $start = $segment_end;
+      }
+      # print STDOUT "  $start-$end ** $rom_segment\n";
+      $result .= $rom_segment;
+   }
+   return $result;
+}
+
+sub quick_romanize {
+   local($this, $s, $lang_code, *ht) = @_;
+
+   my $result = "";
+   my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   while (@chars) {
+      my $found_match_in_table_p = 0;
+      foreach $string_length (reverse(1..4)) {
+	 next if ($string_length-1) > $#chars;
+	 $multi_char_substring = join("", @chars[0..($string_length-1)]);
+	 my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
+	 @mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
+	 if (@mappings) {
+	    my $mapping = $mappings[0];
+	    $result .= $mapping;
+            foreach $_ ((1 .. $string_length)) {
+	       shift @chars;
+	    }
+	    $found_match_in_table_p = 1;
+	    last;
+	 }
+      }
+      unless ($found_match_in_table_p) {
+	 $result .= $chars[0];
+	 shift @chars;
+      }
+   }
+   return $result;
+}
+
+sub char_is_combining_char {
+   local($this, $c, *ht) = @_;
+
+   return 0 unless $c;
+   my $category = $ht{UTF_TO_CAT}->{$c};
+   return 0 unless $category;
+   return $category =~ /^M/;
+}
+
+sub mark_up_string_for_mouse_over {
+   local($this, $s, *ht, $control, *pinyin_ht) = @_;
+
+   $control = "" unless defined($control);
+   $no_ascii_p = ($control =~ /NO-ASCII/);
+   my $result = "";
+   @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   while (@chars) {
+      $char = shift @chars;
+      $numeric = $ht{UTF_TO_NUMERIC}->{$char};
+      $numeric = "" unless defined($numeric);
+      $pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
+      $pic_descr = "" unless defined($pic_descr);
+      $next_char = ($#chars >= 0) ? $chars[0] : "";
+      $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+      if ($no_ascii_p
+       && ($char =~ /^[\x00-\x7F]*$/)
+       && ! $next_char_is_combining_p) {
+	 $result .= $util->guard_html($char);
+      } elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
+	 $unicode = $utf8->utf8_to_unicode($char);
+	 $title = "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode));
+	 $title .= "&#xA;Chinese: $tonal_translit" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
+	 $title .= "&#xA;Number: $numeric" if $numeric =~ /\d/;
+	 $result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
+      } elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
+	 $title = $char_name;
+	 $title .= "&#xA;Number: $numeric" if $numeric =~ /\d/;
+	 $title .= "&#xA;Picture: $pic_descr" if $pic_descr =~ /\S/;
+	 $char_plus = $char;
+	 while ($next_char_is_combining_p) {
+	    # combining marks (Mc:non-spacing, Mc:spacing combining, Me: enclosing)
+	    $next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char};
+	    $title .= "&#xA;+ $next_char_name";
+	    $char = shift @chars;
+	    $char_plus .= $char;
+	    $next_char = ($#chars >= 0) ? $chars[0] : "";
+	    $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+	 }
+	 $result .= "<span title=\"$title\">" . $util->guard_html($char_plus) . "<\/span>";
+	 $result .= "<wbr>" if $char_name =~ /^(FULLWIDTH COLON|FULLWIDTH COMMA|FULLWIDTH RIGHT PARENTHESIS|IDEOGRAPHIC COMMA|IDEOGRAPHIC FULL STOP|RIGHT CORNER BRACKET)$/;
+      } elsif (($unicode = $utf8->utf8_to_unicode($char))
+	    && ($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
+	 $title = "Hangul syllable U+" . (uc sprintf("%04x", $unicode));
+	 $result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
+      } else {
+	 $result .= $util->guard_html($char);
+      }
+   }
+   return $result;
+}
+
+sub romanize_char_at_position_incl_multi {
+   local($this, $i, $lang_code, $output_style, *ht, *chart_ht) = @_;
+
+   my $char = $chart_ht{ORIG_CHAR}->{$i};
+   return "" unless defined($char);
+   my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$char}}; 
+   return $mappings[0] if @mappings;
+   @mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$char}};
+   return $mappings[0] if @mappings;
+   return $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht);
+}
+
+sub romanize_char_at_position {
+   local($this, $i, $lang_code, $output_style, *ht, *chart_ht) = @_;
+
+   my $char = $chart_ht{ORIG_CHAR}->{$i};
+   return "" unless defined($char);
+   return $char if $char =~ /^[\x00-\x7F]$/; # ASCII
+   my $romanization = $ht{UTF_TO_CHAR_ROMANIZATION}->{$char};
+   return $romanization if $romanization;
+   my $char_name = $chart_ht{CHAR_NAME}->{$i};
+   $romanization = $this->romanize_charname($char_name, $lang_code, $output_style, *ht, $char);
+   $ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization}
+      = ($ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization} || 0) + 1
+      unless (length($romanization) < 4) 
+          || ($romanization =~ /\s/)
+          || ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,3}[aeiou]-$/) # Khmer ngo-/nyo-/pho- OK
+          || ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,2}[aeiougw][aeiou]{1,2}$/) # Canadian, Ethiopic syllable OK
+	  || ($romanization =~ /^(allah|bbux|nyaa|nnya|quuv|rrep|shch|shur|syrx)$/i)  # Arabic; Yi; Ethiopic syllable nyaa; Cyrillic letter shcha
+          || (($char_name =~ /^(YI SYLLABLE|VAI SYLLABLE|ETHIOPIC SYLLABLE|CANADIAN SYLLABICS|CANADIAN SYLLABICS CARRIER)\s+(\S+)$/) && (length($romanization) <= 5));
+   # print STDERR "romanize_char_at_position $i $char_name :: $romanization\n" if $char_name =~ /middle/i;
+   return $romanization;
+}
+
+sub romanize_charname {
+   local($this, $char_name, $lang_code, $output_style, *ht, $char) = @_;
+
+   my $cached_result = $ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style};
+   # print STDERR "(C) romanize_charname($char_name): $cached_result\n" if $cached_result && ($char_name =~ /middle/i);
+   return $cached_result if defined($cashed_result);
+   $orig_char_name = $char_name;
+   $char_name =~ s/^.* LETTER\s+([A-Z]+)-\d+$/$1/; # HENTAIGANA LETTER A-3
+   $char_name =~ s/^.* LETTER\s+//;
+   $char_name =~ s/^.* SYLLABLE\s+B\d\d\d\s+//; # Linear B syllables
+   $char_name =~ s/^.* SYLLABLE\s+//;
+   $char_name =~ s/^.* SYLLABICS\s+//;
+   $char_name =~ s/^.* LIGATURE\s+//;
+   $char_name =~ s/^.* VOWEL SIGN\s+//;
+   $char_name =~ s/^.* CONSONANT SIGN\s+//;
+   $char_name =~ s/^.* CONSONANT\s+//;
+   $char_name =~ s/^.* VOWEL\s+//;
+   $char_name =~ s/ WITH .*$//;
+   $char_name =~ s/ WITHOUT .*$//;
+   $char_name =~ s/\s+(ABOVE|AGUNG|BAR|BARREE|BELOW|CEDILLA|CEREK|DIGRAPH|DOACHASHMEE|FINAL FORM|GHUNNA|GOAL|INITIAL FORM|ISOLATED FORM|KAWI|LELET|LELET RASWADI|LONSUM|MAHAPRANA|MEDIAL FORM|MURDA|MURDA MAHAPRANA|REVERSED|ROTUNDA|SASAK|SUNG|TAM|TEDUNG|TYPE ONE|TYPE TWO|WOLOSO)\s*$//;
+   $char_name =~ s/^([A-Z]+)\d+$/$1/; # Linear B syllables etc.
+   foreach $_ ((1 .. 3)) {
+      $char_name =~ s/^.*\b(?:ABKHASIAN|ACADEMY|AFRICAN|AIVILIK|AITON|AKHMIMIC|ALEUT|ALI GALI|ALPAPRAANA|ALTERNATE|ALTERNATIVE|AMBA|ARABIC|ARCHAIC|ASPIRATED|ATHAPASCAN|BASELINE|BLACKLETTER|BARRED|BASHKIR|BERBER|BHATTIPROLU|BIBLE-CREE|BIG|BINOCULAR|BLACKFOOT|BLENDED|BOTTOM|BROAD|BROKEN|CANDRA|CAPITAL|CARRIER|CHILLU|CLOSE|CLOSED|COPTIC|CROSSED|CRYPTOGRAMMIC|CURLED|CURLY|CYRILLIC|DANTAJA|DENTAL|DIALECT-P|DIAERESIZED|DOTLESS|DOUBLE|DOUBLE-STRUCK|EASTERN PWO KAREN|EGYPTOLOGICAL|FARSI|FINAL|FLATTENED|GLOTTAL|GREAT|GREEK|HALF|HIGH|INITIAL|INSULAR|INVERTED|IOTIFIED|JONA|KANTAJA|KASHMIRI|KHAKASSIAN|KHAMTI|KHANDA|KINNA|KIRGHIZ|KOMI|L-SHAPED|LATINATE|LITTLE|LONG|LONG-LEGGED|LOOPED|LOW|MAHAAPRAANA|MALAYALAM|MANCHU|MANDAILING|MATHEMATICAL|MEDIAL|MIDDLE-WELSH|MON|MONOCULAR|MOOSE-CREE|MULTIOCULAR|MUURDHAJA|N-CREE|NARROW|NASKAPI|NDOLE|NEUTRAL|NIKOLSBURG|NORTHERN|NUBIAN|NUNAVIK|NUNAVUT|OJIBWAY|OLD|OPEN|ORKHON|OVERLONG|PALI|PERSIAN|PHARYNGEAL|PRISHTHAMATRA|R-CREE|REDUPLICATION|REVERSED|ROMANIAN|ROUND|ROUNDED|RUDIMENTA|RUMAI PALAUNG|SANSKRIT|SANYAKA|SARA|SAYISI|SCRIPT|SEBATBEIT|SEMISOFT|SGAW KAREN|SHAN|SHARP|SHWE PALAUNG|SHORT|SIBE|SIDEWAYS|SIMALUNGUN|SMALL|SOGDIAN|SOFT|SOUTH-SLAVEY|SOUTHERN|SPIDERY|STIRRUP|STRAIGHT|STRETCHED|SUBSCRIPT|SWASH|TAI LAING|TAILED|TAILLESS|TAALUJA|TH-CREE|TALL|THREE-LEGGED|TURNED|TODO|TOP|TROKUTASTI|TUAREG|UKRAINIAN|UNBLENDED|VISIGOTHIC|VOCALIC|VOICED|VOICELESS|VOLAPUK|WAVY|WESTERN PWO KAREN|WEST-CREE|WESTERN|WIDE|WOODS-CREE|Y-CREE|YENISEI|YIDDISH)\s+//;
+   }
+   $char_name =~ s/\s+(ABOVE|AGUNG|BAR|BARREE|BELOW|CEDILLA|CEREK|DIGRAPH|DOACHASHMEE|FINAL FORM|GHUNNA|GOAL|INITIAL FORM|ISOLATED FORM|KAWI|LELET|LELET RASWADI|LONSUM|MAHAPRANA|MEDIAL FORM|MURDA|MURDA MAHAPRANA|REVERSED|ROTUNDA|SASAK|SUNG|TAM|TEDUNG|TYPE ONE|TYPE TWO|WOLOSO)\s*$//;
+   if ($char_name =~ /THAI CHARACTER/) {
+      $char_name =~ s/^THAI CHARACTER\s+//;
+      if ($char =~ /^\xE0\xB8[\x81-\xAE]/) {
+	 # Thai consonants
+	 $char_name =~ s/^([^AEIOU]*).*/$1/i;
+      } elsif ($char_name =~ /^SARA [AEIOU]/) {
+	 # Thai vowels
+	 $char_name =~ s/^SARA\s+//;
+      } else {
+	 $char_name = $char;
+      }
+   }
+   if ($orig_char_name =~ /(HIRAGANA LETTER|KATAKANA LETTER|SYLLABLE|LIGATURE)/) {
+      $char_name = lc $char_name;
+   } elsif ($char_name =~ /\b(ANUSVARA|ANUSVARAYA|NIKAHIT|SIGN BINDI|TIPPI)\b/) {
+      $char_name = "+m";
+   } elsif ($char_name =~ /\bSCHWA\b/) {
+      $char_name = "e";
+   } elsif ($char_name =~ /\bIOTA\b/) {
+      $char_name = "i";
+   } elsif ($char_name =~ /\s/) {
+   } elsif ($orig_char_name =~ /KHMER LETTER/) {
+      $char_name .= "-";
+   } elsif ($orig_char_name =~ /CHEROKEE LETTER/) {
+      # use whole letter as is
+   } elsif ($orig_char_name =~ /KHMER INDEPENDENT VOWEL/) {
+      $char_name =~ s/q//;
+   } elsif ($orig_char_name =~ /LETTER/) {
+      $char_name =~ s/^[AEIOU]+([^AEIOU]+)$/$1/i;
+      $char_name =~ s/^([^-AEIOUY]+)[AEIOU].*/$1/i;
+      $char_name =~ s/^(Y)[AEIOU].*/$1/i if $orig_char_name =~ /\b(?:BENGALI|DEVANAGARI|GURMUKHI|GUJARATI|KANNADA|MALAYALAM|MODI|MYANMAR|ORIYA|TAMIL|TELUGU|TIBETAN)\b.*\bLETTER YA\b/;
+      $char_name =~ s/^(Y[AEIOU]+)[^AEIOU].*$/$1/i;
+      $char_name =~ s/^([AEIOU]+)[^AEIOU]+[AEIOU].*/$1/i;
+   }
+
+   my $result = ($orig_char_name =~ /\bCAPITAL\b/) ? (uc $char_name) : (lc $char_name);
+   # print STDERR "(R) romanize_charname($orig_char_name): $result\n" if $orig_char_name =~ /middle/i;
+   $ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style} = $result;
+   return $result;
+}
+
+sub assemble_numbers_in_chart {
+   local($this, *chart_ht, $line_number) = @_;
+
+   foreach $start (sort { $a <=> $b } keys %{$chart_ht{COMPLEX_NUMERIC_START_END}}) {
+      my $end = $chart_ht{COMPLEX_NUMERIC_START_END}->{$start};
+      my @numbers = ();
+      foreach $i (($start .. ($end-1))) {
+         my $orig_char = $chart_ht{ORIG_CHAR}->{$i};
+         my $node_id = $this->get_node_for_span_with_slot($i, $i+1, "numeric-value", *chart_id);
+	 if (defined($node_id)) {
+	    my $number = $chart_ht{NODE_ROMAN}->{$node_id};
+	    if (defined($number)) {
+               push(@numbers, $number);
+	    } elsif ($orig_char =~ /^[.,]$/) { # decimal point, comma separator
+	       push(@numbers, $orig_char);
+	    } else {
+	       print STDERR "Found no romanization for node_id $node_id ($i-" . ($i+1) . ") in assemble_numbers_in_chart\n" if $verbosePM;
+	    }
+	 } else {
+	    print STDERR "Found no node_id for span $i-" . ($i+1) . " in assemble_numbers_in_chart\n" if $verbosePM;
+	 }
+      }
+      my $complex_number = $this->assemble_number(join("\xC2\xB7", @numbers), $line_number);
+      # print STDERR "assemble_numbers_in_chart l.$line_number $start-$end $complex_number (@numbers)\n";
+      $this->add_node($complex_number, $start, $end, *chart_ht, "", "complex-number");
+   }
+}
+
+sub assemble_number {
+   local($this, $s, $line_number) = @_;
+   # e.g. 10 9 100 7 10 8 = 1978
+
+   my $middot = "\xC2\xB7";
+   my @tokens = split(/$middot/, $s); # middle dot U+00B7
+   my $i = 0;
+   my @orig_tokens = @tokens;
+
+   # assemble single digit numbers, e.g. 1 7 5 -> 175
+   while ($i < $#tokens) {
+      if ($tokens[$i] =~ /^\d$/) {
+         my $j = $i+1;
+	 while (($j <= $#tokens) && ($tokens[$j] =~ /^[0-9.,]$/)) {
+	    $j++;
+	 }
+	 $j--;
+	 if ($j>$i) {
+	    my $new_token = join("", @tokens[$i .. $j]);
+	    $new_token =~ s/,//g;
+	    splice(@tokens, $i, $j-$i+1, $new_token);
+	 }
+      }
+      $i++;
+   }
+
+   foreach $power ((10, 100, 1000, 10000, 100000, 1000000, 100000000, 1000000000, 1000000000000)) {
+      for (my $i=0; $i <= $#tokens; $i++) {
+	 if ($tokens[$i] == $power) {
+            if (($i > 0) && ($tokens[($i-1)] < $power)) {
+	       splice(@tokens, $i-1, 2, ($tokens[($i-1)] * $tokens[$i]));
+	       $i--;
+               if (($i < $#tokens) && ($tokens[($i+1)] < $power)) {
+	          splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
+	          $i--;
+	       }
+	    }
+	 } 
+	 # 400 30 (e.g. Egyptian)
+	 my $gen_pattern = $power;
+         $gen_pattern =~ s/^1/\[1-9\]/;
+         if (($tokens[$i] =~ /^$gen_pattern$/) && ($i < $#tokens) && ($tokens[($i+1)] < $power)) {
+	    splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
+	    $i--;
+	 }
+      }
+      last if $#tokens == 0;
+   }
+   my $result = join($middot, @tokens);
+   if ($verbosePM) {
+      my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-number-log.txt";
+      $util->append_to_file($logfile, "$s -> $result\n") if -r $logfile;
+      # print STDERR "  assemble number l.$line_number @orig_tokens -> $result\n" if $line_number == 43;
+   }
+   return $result;
+}
+
+1;
+

uroman/lib/NLP/UTF8.pm

@@ -0,0 +1,1404 @@
+################################################################
+#                                                              #
+# UTF8                                                         #
+#                                                              #
+################################################################
+
+package NLP::UTF8;
+
+use NLP::utilities;
+$util = NLP::utilities;
+
+%empty_ht = ();
+
+sub new {
+   local($caller) = @_;
+
+   my $object = {};
+   my $class = ref( $caller ) || $caller;
+   bless($object, $class);
+   return $object;
+}
+
+sub unicode_string2string {
+# input: string that might contain unicode sequences such as "U+0627"
+# output: string in pure utf-8
+   local($caller,$s) = @_;
+
+   my $pre; 
+   my $unicode; 
+   my $post; 
+   my $r1; 
+   my $r2; 
+   my $r3;
+
+   ($pre,$unicode,$post) = ($s =~ /^(.*)(?:U\+|\\u)([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])(.*)$/);
+   return $s unless defined($post);
+   $r1 = $caller->unicode_string2string($pre);
+   $r2 = $caller->unicode_hex_string2string($unicode);
+   $r3 = $caller->unicode_string2string($post);
+   $result = $r1 . $r2 . $r3;
+   return $result;
+}
+
+sub unicode_hex_string2string {
+# input: "0627" (interpreted as hex code)
+# output: utf-8 string for Arabic letter alef
+   local($caller,$unicode) = @_;
+   return "" unless defined($unicode);
+   my $d = hex($unicode);
+   return $caller->unicode2string($d);
+}
+
+sub unicode2string {
+# input: non-neg integer, e.g. 0x627
+# output: utf-8 string for Arabic letter alef
+   local($caller,$d) = @_;
+   return "" unless defined($d) && $d >= 0;
+   return sprintf("%c",$d) if $d <= 0x7F;
+
+   my $lastbyte1 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c",$d | 0xC0, $lastbyte1) if $d <= 0x1F;
+
+   my $lastbyte2 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c",$d | 0xE0, $lastbyte2, $lastbyte1) if $d <= 0xF;
+
+   my $lastbyte3 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c%c",$d | 0xF0, $lastbyte3, $lastbyte2, $lastbyte1) if $d <= 0x7;
+
+   my $lastbyte4 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c%c%c",$d | 0xF8, $lastbyte4, $lastbyte3, $lastbyte2, $lastbyte1) if $d <= 0x3;
+
+   my $lastbyte5 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c%c%c%c",$d | 0xFC, $lastbyte5, $lastbyte4, $lastbyte3, $lastbyte2, $lastbyte1) if $d <= 0x1;
+   return ""; # bad input
+}
+
+sub html2utf8 {
+   local($caller, $string) = @_;
+
+   return $string unless $string =~ /\&\#\d{3,5};/;
+
+   my $prev = "";
+   my $s = $string;
+   while ($s ne $prev) {
+      $prev = $s;
+      ($pre,$d,$post) = ($s =~ /^(.*)\&\#(\d+);(.*)$/);
+      if (defined($d) && ((($d >= 160) && ($d <= 255))
+                       || (($d >= 1500) && ($d <= 1699))
+                       || (($d >= 19968) && ($d <= 40879)))) {
+         $html_code = "\&\#" . $d . ";";
+         $utf8_code = $caller->unicode2string($d);
+         $s =~ s/$html_code/$utf8_code/;
+      }
+   }
+   return $s;
+}
+
+sub xhtml2utf8 {
+   local($caller, $string) = @_;
+
+   return $string unless $string =~ /\&\#x[0-9a-fA-F]{2,5};/;
+
+   my $prev = "";
+   my $s = $string;
+   while ($s ne $prev) {
+      $prev = $s;
+      if (($pre, $html_code, $x, $post) = ($s =~ /^(.*)(\&\#x([0-9a-fA-F]{2,5});)(.*)$/)) {
+         $utf8_code = $caller->unicode_hex_string2string($x);
+         $s =~ s/$html_code/$utf8_code/;
+      }
+   }
+   return $s;
+}
+
+sub utf8_marker {
+   return sprintf("%c%c%c\n", 0xEF, 0xBB, 0xBF);
+}
+
+sub enforcer {
+# input: string that might not conform to utf-8
+# output: string in pure utf-8, with a few "smart replacements" and possibly "?"
+   local($caller,$s,$no_repair) = @_;
+
+   my $ascii;
+   my $utf8;
+   my $rest;
+
+   return $s if $s =~ /^[\x00-\x7F]*$/;
+
+   $no_repair = 0 unless defined($no_repair);
+   $orig = $s;
+   $result = "";
+
+   while ($s ne "") {
+      ($ascii,$rest) = ($s =~ /^([\x00-\x7F]+)(.*)$/);
+      if (defined($ascii)) {
+	 $result .= $ascii;
+	 $s = $rest;
+	 next;
+      }
+      ($utf8,$rest) = ($s =~ /^([\xC0-\xDF][\x80-\xBF])(.*)$/);
+      ($utf8,$rest) = ($s =~ /^([\xE0-\xEF][\x80-\xBF][\x80-\xBF])(.*)$/) 
+	 unless defined($rest);
+      ($utf8,$rest) = ($s =~ /^([\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF])(.*)$/)
+	 unless defined($rest);
+      ($utf8,$rest) = ($s =~ /^([\xF8-\xFB][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF])(.*)$/) 
+	 unless defined($rest);
+      if (defined($utf8)) {
+	 $result .= $utf8;
+	 $s = $rest;
+	 next;
+      }
+      ($c,$rest) = ($s =~ /^(.)(.*)$/);
+      if (defined($c)) {
+	 if    ($no_repair)   { $result .= "?"; }
+	 elsif ($c =~ /\x85/) { $result .= "..."; }
+	 elsif ($c =~ /\x91/) { $result .= "'"; }
+	 elsif ($c =~ /\x92/) { $result .= "'"; }
+	 elsif ($c =~ /\x93/) { $result .= $caller->unicode2string(0x201C); }
+	 elsif ($c =~ /\x94/) { $result .= $caller->unicode2string(0x201D); }
+	 elsif ($c =~ /[\xC0-\xFF]/) {
+	    $c2 = $c;
+	    $c2 =~ tr/[\xC0-\xFF]/[\x80-\xBF]/;
+	    $result .= "\xC3$c2";
+	 } else {
+	    $result .= "?";
+	 }
+	 $s = $rest;
+	 next;
+      }
+      $s = "";
+   }
+   $result .= "\n" if ($orig =~ /\n$/) && ! ($result =~ /\n$/);
+   return $result;
+}
+
+sub split_into_utf8_characters {
+# input: utf8 string
+# output: list of sub-strings, each representing a utf8 character
+   local($caller,$string,$group_control, *ht) = @_;
+
+   @characters = ();
+   $end_of_token_p_string = "";
+   $skipped_bytes = "";
+   $group_control = "" unless defined($group_control);
+   $group_ascii_numbers = ($group_control =~ /ASCII numbers/);
+   $group_ascii_spaces = ($group_control =~ /ASCII spaces/);
+   $group_ascii_punct = ($group_control =~ /ASCII punct/);
+   $group_ascii_chars = ($group_control =~ /ASCII chars/);
+   $group_xml_chars = ($group_control =~ /XML chars/);
+   $group_xml_tags = ($group_control =~ /XML tags/);
+   $return_only_chars = ($group_control =~ /return only chars/);
+   $return_trailing_whitespaces = ($group_control =~ /return trailing whitespaces/);
+   if ($group_control =~ /ASCII all/) {
+      $group_ascii_numbers = 1;
+      $group_ascii_spaces = 1;
+      $group_ascii_chars = 1;
+      $group_ascii_punct = 1;
+   }
+   if ($group_control =~ /(XML chars and tags|XML tags and chars)/) {
+      $group_xml_chars = 1;
+      $group_xml_tags = 1;
+   }
+   $orig_string = $string;
+   $string .= " ";
+   while ($string =~ /\S/) {
+      # one-character UTF-8 = ASCII
+      if ($string =~ /^[\x00-\x7F]/) {
+	 if ($group_xml_chars
+	  && (($dec_unicode, $rest) = ($string =~ /^&#(\d+);(.*)$/s))
+	  && ($utf8_char = $caller->unicode2string($dec_unicode))) {
+	    push(@characters, $utf8_char);
+	    $string = $rest;
+	 } elsif ($group_xml_chars
+	  && (($hex_unicode, $rest) = ($string =~ /^&#x([0-9a-f]{1,6});(.*)$/is))
+	  && ($utf8_char = $caller->unicode_hex_string2string($hex_unicode))) {
+	    push(@characters, $utf8_char);
+	    $string = $rest;
+	 } elsif ($group_xml_chars
+	  && (($html_entity_name, $rest) = ($string =~ /^&([a-z]{1,6});(.*)$/is))
+	  && ($dec_unicode = $ht{HTML_ENTITY_NAME_TO_DECUNICODE}->{$html_entity_name})
+	  && ($utf8_char = $caller->unicode2string($dec_unicode))
+	  ) {
+	    push(@characters, $utf8_char);
+	    $string = $rest;
+	 } elsif ($group_xml_tags
+	       && (($tag, $rest) = ($string =~ /^(<\/?[a-zA-Z][-_:a-zA-Z0-9]*(\s+[a-zA-Z][-_:a-zA-Z0-9]*=\"[^"]*\")*\s*\/?>)(.*)$/s))) {
+            push(@characters, $tag);
+	    $string = $rest;
+	 } elsif ($group_ascii_numbers && ($string =~ /^[12]\d\d\d\.[01]?\d.[0-3]?\d([^0-9].*)?$/)) {
+	    ($date) = ($string =~ /^(\d\d\d\d\.\d?\d.\d?\d)([^0-9].*)?$/);
+	    push(@characters,$date);
+	    $string = substr($string, length($date));
+	 } elsif ($group_ascii_numbers && ($string =~ /^\d/)) {
+	    ($number) = ($string =~ /^(\d+(,\d\d\d)*(\.\d+)?)/);
+	    push(@characters,$number);
+	    $string = substr($string, length($number));
+	 } elsif ($group_ascii_spaces && ($string =~ /^(\s+)/)) {
+	    ($space) = ($string =~ /^(\s+)/);
+	    $string = substr($string, length($space));
+	 } elsif ($group_ascii_punct && (($punct_seq) = ($string =~ /^(-+|\.+|[:,%()"])/))) {
+	    push(@characters,$punct_seq);
+	    $string = substr($string, length($punct_seq));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^(\$[A-Z]*|[A-Z]{1,3}\$)/))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && (($abbrev) = ($string =~ /^((?:Jan|Feb|Febr|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec|Mr|Mrs|Dr|a.m|p.m)\.)/))) {
+	    push(@characters,$abbrev);
+	    $string = substr($string, length($abbrev));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^(second|minute|hour|day|week|month|year|inch|foot|yard|meter|kilometer|mile)-(?:long|old)/i))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^(zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|trillion)-/i))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^([a-zA-Z]+)(?:[ ,;%?|()"]|'s |' |\. |\d+[:hms][0-9 ])/))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && ($string =~ /^([\x21-\x27\x2A-\x7E]+)/)) { # exclude ()
+	    ($ascii) = ($string =~ /^([\x21-\x27\x2A-\x7E]+)/);  # ASCII black-characters
+	    push(@characters,$ascii);
+	    $string = substr($string, length($ascii));
+	 } elsif ($group_ascii_chars && ($string =~ /^([\x21-\x7E]+)/)) {
+	    ($ascii) = ($string =~ /^([\x21-\x7E]+)/);  # ASCII black-characters
+	    push(@characters,$ascii);
+	    $string = substr($string, length($ascii));
+	 } elsif ($group_ascii_chars && ($string =~ /^([\x00-\x7F]+)/)) {
+	    ($ascii) = ($string =~ /^([\x00-\x7F]+)/);
+	    push(@characters,$ascii);
+	    $string = substr($string, length($ascii));
+	 } else {
+	    push(@characters,substr($string, 0, 1));
+	    $string = substr($string, 1);
+	 }
+
+      # two-character UTF-8
+      } elsif ($string =~ /^[\xC0-\xDF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 2));
+	 $string = substr($string, 2);
+
+      # three-character UTF-8
+      } elsif ($string =~ /^[\xE0-\xEF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 3));
+	 $string = substr($string, 3);
+
+      # four-character UTF-8
+      } elsif ($string =~ /^[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 4));
+	 $string = substr($string, 4);
+
+      # five-character UTF-8
+      } elsif ($string =~ /^[\xF8-\xFB][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 5));
+	 $string = substr($string, 5);
+
+      # six-character UTF-8
+      } elsif ($string =~ /^[\xFC-\xFD][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 6));
+	 $string = substr($string, 6);
+
+      # not a UTF-8 character
+      } else {
+         $skipped_bytes .= substr($string, 0, 1);
+	 $string = substr($string, 1);
+      }
+      
+      $end_of_token_p_string .= ($string =~ /^\S/) ? "0" : "1" 
+	 if $#characters >= length($end_of_token_p_string);
+   }
+   $string =~ s/ $//; # remove previously added space, but keep original spaces
+   if ($return_trailing_whitespaces) {
+      while ($string =~ /^[ \t]/) {
+         push(@characters,substr($string, 0, 1));
+         $string = substr($string, 1);
+      }
+      push(@characters, "\n") if $orig_string =~ /\n$/;
+   }
+   return ($return_only_chars) ? @characters : ($skipped_bytes, $end_of_token_p_string, @characters);
+}
+
+sub max_substring_info {
+   local($caller,$s1,$s2,$info_type) = @_;
+
+   ($skipped_bytes1, $end_of_token_p_string1, @char_list1) = $caller->split_into_utf8_characters($s1, "", *empty_ht);
+   ($skipped_bytes2, $end_of_token_p_string2, @char_list2) = $caller->split_into_utf8_characters($s2, "", *empty_ht);
+   return 0 if $skipped_bytes1 || $skipped_bytes2;
+
+   $best_substring_start1 = 0;
+   $best_substring_start2 = 0;
+   $best_substring_length = 0;
+
+   foreach $start_pos2 ((0 .. $#char_list2)) {
+      last if $start_pos2 + $best_substring_length > $#char_list2;
+      foreach $start_pos1 ((0 .. $#char_list1)) {
+         last if $start_pos1 + $best_substring_length > $#char_list1;
+         $matching_length = 0;
+         while (($start_pos1 + $matching_length <= $#char_list1)
+	     && ($start_pos2 + $matching_length <= $#char_list2)
+	     && ($char_list1[$start_pos1+$matching_length] eq $char_list2[$start_pos2+$matching_length])) {
+	    $matching_length++;
+	 }
+	 if ($matching_length > $best_substring_length) {
+	    $best_substring_length = $matching_length;
+	    $best_substring_start1 = $start_pos1;
+	    $best_substring_start2 = $start_pos2;
+	 }
+      }
+   }
+   if ($info_type =~ /^max-ratio1$/) {
+      $length1 = $#char_list1 + 1;
+      return ($length1 > 0) ? ($best_substring_length / $length1) : 0;
+   } elsif ($info_type =~ /^max-ratio2$/) {
+      $length2 = $#char_list2 + 1;
+      return ($length2 > 0) ? ($best_substring_length / $length2) : 0;
+   } elsif ($info_type =~ /^substring$/) {
+      return join("", @char_list1[$best_substring_start1 .. $best_substring_start1+$best_substring_length-1]);
+   } else {
+      $length1 = $#char_list1 + 1;
+      $length2 = $#char_list2 + 1;
+      $info = "s1=$s1;s2=$s2";
+      $info .= ";best_substring_length=$best_substring_length";
+      $info .= ";best_substring_start1=$best_substring_start1";
+      $info .= ";best_substring_start2=$best_substring_start2";
+      $info .= ";length1=$length1";
+      $info .= ";length2=$length2";
+      return $info;
+   }
+}
+
+sub n_shared_chars_at_start {
+   local($caller,$s1,$s2) = @_;
+
+   my $n = 0;
+   while (($s1 ne "") && ($s2 ne "")) {
+      ($c1, $rest1) = ($s1 =~ /^(.[\x80-\xBF]*)(.*)$/);
+      ($c2, $rest2) = ($s2 =~ /^(.[\x80-\xBF]*)(.*)$/);
+      if ($c1 eq $c2) {
+	 $n++;
+	 $s1 = $rest1;
+	 $s2 = $rest2;
+      } else {
+	 last;
+      }
+   }
+   return $n;
+}
+
+sub char_length {
+   local($caller,$string,$byte_offset) = @_;
+
+   my $char = ($byte_offset) ? substr($string, $byte_offset) : $string;
+   return 1 if $char =~ /^[\x00-\x7F]/;
+   return 2 if $char =~ /^[\xC0-\xDF]/;
+   return 3 if $char =~ /^[\xE0-\xEF]/;
+   return 4 if $char =~ /^[\xF0-\xF7]/;
+   return 5 if $char =~ /^[\xF8-\xFB]/;
+   return 6 if $char =~ /^[\xFC-\xFD]/;
+   return 0;
+}
+
+sub length_in_utf8_chars {
+   local($caller,$s) = @_;
+
+   $s =~ s/[\x80-\xBF]//g;
+   $s =~ s/[\x00-\x7F\xC0-\xFF]/c/g;
+   return length($s);
+}
+
+sub byte_length_of_n_chars {
+   local($caller,$char_length,$string,$byte_offset,$undef_return_value) = @_;
+
+   $byte_offset = 0 unless defined($byte_offset);
+   $undef_return_value = -1 unless defined($undef_return_value);
+   my $result = 0;
+   my $len;
+   foreach $i ((1 .. $char_length)) {
+      $len = $caller->char_length($string,($byte_offset+$result));
+      return $undef_return_value unless $len;
+      $result += $len;
+   }
+   return $result;
+}
+
+sub replace_non_ASCII_bytes {
+   local($caller,$string,$replacement) = @_;
+
+   $replacement = "HEX" unless defined($replacement);
+   if ($replacement =~ /^(Unicode|U\+4|\\u|HEX)$/) {
+      $new_string = "";
+      while (($pre,$utf8_char, $post) = ($string =~ /^([\x09\x0A\x20-\x7E]*)([\x00-\x08\x0B-\x1F\x7F]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF]|[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF]|[\xF8-\xFF][\x80-\xBF]+|[\x80-\xBF])(.*)$/s)) {
+	 if ($replacement =~ /Unicode/) {
+	    $new_string .= $pre . "<U" . (uc $caller->utf8_to_unicode($utf8_char)) . ">";
+	 } elsif ($replacement =~ /\\u/) {
+	    $new_string .= $pre . "\\u" . (uc sprintf("%04x", $caller->utf8_to_unicode($utf8_char)));
+	 } elsif ($replacement =~ /U\+4/) {
+	    $new_string .= $pre . "<U+" . (uc $caller->utf8_to_4hex_unicode($utf8_char)) . ">";
+	 } else {
+	    $new_string .= $pre . "<HEX-" . $caller->utf8_to_hex($utf8_char) . ">";
+	 }
+	 $string = $post;
+      }
+      $new_string .= $string;
+   } else {
+      $new_string = $string;
+      $new_string =~ s/[\x80-\xFF]/$replacement/g;
+   }
+   return $new_string;
+}
+
+sub valid_utf8_string_p {
+   local($caller,$string) = @_;
+
+   return $string =~ /^(?:[\x09\x0A\x20-\x7E]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF]|[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF])*$/;
+}
+
+sub valid_utf8_string_incl_ascii_control_p {
+   local($caller,$string) = @_;
+
+   return $string =~ /^(?:[\x00-\x7F]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF]|[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF])*$/;
+}
+
+sub utf8_to_hex {
+   local($caller,$s) = @_;
+
+   $hex = "";
+   foreach $i ((0 .. length($s)-1)) {
+      $hex .= uc sprintf("%2.2x",ord(substr($s, $i, 1)));
+   }
+   return $hex;
+}
+
+sub hex_to_utf8 {
+   local($caller,$s) = @_;
+   # surface string \xE2\x80\xBA to UTF8
+
+   my $utf8 = "";
+   while (($hex, $rest) = ($s =~ /^(?:\\x)?([0-9A-Fa-f]{2,2})(.*)$/)) {
+      $utf8 .= sprintf("%c", hex($hex));
+      $s = $rest;
+   }
+   return $utf8;
+}
+
+sub utf8_to_4hex_unicode {
+   local($caller,$s) = @_;
+
+   return sprintf("%4.4x", $caller->utf8_to_unicode($s));
+}
+
+sub utf8_to_unicode {
+   local($caller,$s) = @_;
+
+   $unicode = 0;
+   foreach $i ((0 .. length($s)-1)) {
+      $c = substr($s, $i, 1);
+      if ($c =~ /^[\x80-\xBF]$/) {
+	 $unicode = $unicode * 64 + (ord($c) & 0x3F);
+      } elsif ($c =~ /^[\xC0-\xDF]$/) {
+	 $unicode = $unicode * 32 + (ord($c) & 0x1F);
+      } elsif ($c =~ /^[\xE0-\xEF]$/) {
+	 $unicode = $unicode * 16 + (ord($c) & 0x0F);
+      } elsif ($c =~ /^[\xF0-\xF7]$/) {
+	 $unicode = $unicode * 8 + (ord($c) & 0x07);
+      } elsif ($c =~ /^[\xF8-\xFB]$/) {
+	 $unicode = $unicode * 4 + (ord($c) & 0x03);
+      } elsif ($c =~ /^[\xFC-\xFD]$/) {
+	 $unicode = $unicode * 2 + (ord($c) & 0x01);
+      }
+   }
+   return $unicode;
+}
+
+sub charhex {
+   local($caller,$string) = @_;
+
+   my $result = "";
+   while ($string ne "") {
+      $char = substr($string, 0, 1);
+      $string = substr($string, 1);
+      if ($char =~ /^[ -~]$/) {
+         $result .= $char;
+      } else {
+	 $hex = sprintf("%2.2x",ord($char));
+	 $hex =~ tr/a-f/A-F/;
+         $result .= "<HEX-$hex>";
+      }
+   }
+   return $result;
+}
+
+sub windows1252_to_utf8 {
+   local($caller,$s, $norm_to_ascii_p, $preserve_potential_utf8s_p) = @_;
+
+   return $s if $s =~ /^[\x00-\x7F]*$/; # all ASCII
+
+   $norm_to_ascii_p = 1 unless defined($norm_to_ascii_p);
+   $preserve_potential_utf8s_p = 1 unless defined($preserve_potential_utf8s_p);
+   my $result = "";
+   my $c = "";
+   while ($s ne "") {
+      $n_bytes = 1;
+      if ($s =~ /^[\x00-\x7F]/) {
+	 $result .= substr($s, 0, 1);  # ASCII
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xC0-\xDF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 2);  # valid 2-byte UTF8
+         $n_bytes = 2;
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xE0-\xEF][\x80-\xBF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 3);  # valid 3-byte UTF8
+         $n_bytes = 3;
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 4);  # valid 4-byte UTF8
+         $n_bytes = 4;
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xF8-\xFB][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 5);  # valid 5-byte UTF8
+         $n_bytes = 5;
+      } elsif ($s =~ /^[\xA0-\xBF]/) {
+	 $c = substr($s, 0, 1);
+	 $result .= "\xC2$c";
+      } elsif ($s =~ /^[\xC0-\xFF]/) {
+	 $c = substr($s, 0, 1);
+	 $c =~ tr/[\xC0-\xFF]/[\x80-\xBF]/;
+	 $result .= "\xC3$c";
+      } elsif ($s =~ /^\x80/) {
+	 $result .= "\xE2\x82\xAC";  # Euro sign
+      } elsif ($s =~ /^\x82/) {
+	 $result .= "\xE2\x80\x9A";  # single low quotation mark
+      } elsif ($s =~ /^\x83/) {
+	 $result .= "\xC6\x92";      # Latin small letter f with hook
+      } elsif ($s =~ /^\x84/) {
+	 $result .= "\xE2\x80\x9E";  # double low quotation mark
+      } elsif ($s =~ /^\x85/) {
+	 $result .= ($norm_to_ascii_p) ? "..." : "\xE2\x80\xA6";  # horizontal ellipsis (three dots)
+      } elsif ($s =~ /^\x86/) {
+	 $result .= "\xE2\x80\xA0";  # dagger
+      } elsif ($s =~ /^\x87/) {
+	 $result .= "\xE2\x80\xA1";  # double dagger
+      } elsif ($s =~ /^\x88/) {
+	 $result .= "\xCB\x86";      # circumflex
+      } elsif ($s =~ /^\x89/) {
+	 $result .= "\xE2\x80\xB0";  # per mille sign
+      } elsif ($s =~ /^\x8A/) {
+	 $result .= "\xC5\xA0";      # Latin capital letter S with caron
+      } elsif ($s =~ /^\x8B/) {
+	 $result .= "\xE2\x80\xB9";  # single left-pointing angle quotation mark
+      } elsif ($s =~ /^\x8C/) {
+	 $result .= "\xC5\x92";      # OE ligature
+      } elsif ($s =~ /^\x8E/) {
+	 $result .= "\xC5\xBD";      # Latin capital letter Z with caron
+      } elsif ($s =~ /^\x91/) {
+	 $result .= ($norm_to_ascii_p) ? "`" : "\xE2\x80\x98";  # left single quotation mark
+      } elsif ($s =~ /^\x92/) {
+	 $result .= ($norm_to_ascii_p) ? "'" : "\xE2\x80\x99";  # right single quotation mark
+      } elsif ($s =~ /^\x93/) {
+	 $result .= "\xE2\x80\x9C";  # left double quotation mark
+      } elsif ($s =~ /^\x94/) {
+	 $result .= "\xE2\x80\x9D";  # right double quotation mark
+      } elsif ($s =~ /^\x95/) {
+	 $result .= "\xE2\x80\xA2";  # bullet
+      } elsif ($s =~ /^\x96/) {
+	 $result .= ($norm_to_ascii_p) ? "-" : "\xE2\x80\x93";  # n dash
+      } elsif ($s =~ /^\x97/) {
+	 $result .= ($norm_to_ascii_p) ? "-" : "\xE2\x80\x94";  # m dash
+      } elsif ($s =~ /^\x98/) {
+	 $result .= ($norm_to_ascii_p) ? "~" : "\xCB\x9C";      # small tilde
+      } elsif ($s =~ /^\x99/) {
+	 $result .= "\xE2\x84\xA2";  # trade mark sign
+      } elsif ($s =~ /^\x9A/) {
+	 $result .= "\xC5\xA1";      # Latin small letter s with caron
+      } elsif ($s =~ /^\x9B/) {
+	 $result .= "\xE2\x80\xBA";  # single right-pointing angle quotation mark
+      } elsif ($s =~ /^\x9C/) {
+	 $result .= "\xC5\x93";      # oe ligature
+      } elsif ($s =~ /^\x9E/) {
+	 $result .= "\xC5\xBE";      # Latin small letter z with caron
+      } elsif ($s =~ /^\x9F/) {
+	 $result .= "\xC5\xB8";      # Latin capital letter Y with diaeresis
+      } else {
+	 $result .= "?";
+      }
+      $s = substr($s, $n_bytes);
+   }
+   return $result;
+}
+
+sub delete_weird_stuff {
+   local($caller, $s) = @_;
+
+   # delete control chacters (except tab and linefeed), zero-width characters, byte order mark,
+   # directional marks, join marks, variation selectors, Arabic tatweel
+   $s =~ s/([\x00-\x08\x0B-\x1F\x7F]|\xC2[\x80-\x9F]|\xD9\x80|\xE2\x80[\x8B-\x8F]|\xEF\xB8[\x80-\x8F]|\xEF\xBB\xBF|\xF3\xA0[\x84-\x87][\x80-\xBF])//g;
+   return $s;
+}
+
+sub number_of_utf8_character {
+   local($caller, $s) = @_;
+
+   $s2 = $s;
+   $s2 =~ s/[\x80-\xBF]//g;
+   return length($s2);
+}
+
+sub cap_letter_reg_exp {
+   # includes A-Z and other Latin-based capital letters with accents, umlauts and other decorations etc.
+   return "[A-Z]|\xC3[\x80-\x96\x98-\x9E]|\xC4[\x80\x82\x84\x86\x88\x8A\x8C\x8E\x90\x94\x964\x98\x9A\x9C\x9E\xA0\xA2\xA4\xA6\xA8\xAA\xAC\xAE\xB0\xB2\xB4\xB6\xB9\xBB\xBD\xBF]|\xC5[\x81\x83\x85\x87\x8A\x8C\x8E\x90\x92\x96\x98\x9A\x9C\x9E\xA0\xA2\xA4\xA6\xA8\xAA\xAC\xB0\xB2\xB4\xB6\xB8\xB9\xBB\xBD]";
+}
+
+sub regex_extended_case_expansion {
+   local($caller, $s) = @_;
+
+   if ($s =~ /\xC3/) {
+   $s =~ s/\xC3\xA0/\xC3\[\x80\xA0\]/g;
+   $s =~ s/\xC3\xA1/\xC3\[\x81\xA1\]/g;
+   $s =~ s/\xC3\xA2/\xC3\[\x82\xA2\]/g;
+   $s =~ s/\xC3\xA3/\xC3\[\x83\xA3\]/g;
+   $s =~ s/\xC3\xA4/\xC3\[\x84\xA4\]/g;
+   $s =~ s/\xC3\xA5/\xC3\[\x85\xA5\]/g;
+   $s =~ s/\xC3\xA6/\xC3\[\x86\xA6\]/g;
+   $s =~ s/\xC3\xA7/\xC3\[\x87\xA7\]/g;
+   $s =~ s/\xC3\xA8/\xC3\[\x88\xA8\]/g;
+   $s =~ s/\xC3\xA9/\xC3\[\x89\xA9\]/g;
+   $s =~ s/\xC3\xAA/\xC3\[\x8A\xAA\]/g;
+   $s =~ s/\xC3\xAB/\xC3\[\x8B\xAB\]/g;
+   $s =~ s/\xC3\xAC/\xC3\[\x8C\xAC\]/g;
+   $s =~ s/\xC3\xAD/\xC3\[\x8D\xAD\]/g;
+   $s =~ s/\xC3\xAE/\xC3\[\x8E\xAE\]/g;
+   $s =~ s/\xC3\xAF/\xC3\[\x8F\xAF\]/g;
+   $s =~ s/\xC3\xB0/\xC3\[\x90\xB0\]/g;
+   $s =~ s/\xC3\xB1/\xC3\[\x91\xB1\]/g;
+   $s =~ s/\xC3\xB2/\xC3\[\x92\xB2\]/g;
+   $s =~ s/\xC3\xB3/\xC3\[\x93\xB3\]/g;
+   $s =~ s/\xC3\xB4/\xC3\[\x94\xB4\]/g;
+   $s =~ s/\xC3\xB5/\xC3\[\x95\xB5\]/g;
+   $s =~ s/\xC3\xB6/\xC3\[\x96\xB6\]/g;
+   $s =~ s/\xC3\xB8/\xC3\[\x98\xB8\]/g;
+   $s =~ s/\xC3\xB9/\xC3\[\x99\xB9\]/g;
+   $s =~ s/\xC3\xBA/\xC3\[\x9A\xBA\]/g;
+   $s =~ s/\xC3\xBB/\xC3\[\x9B\xBB\]/g;
+   $s =~ s/\xC3\xBC/\xC3\[\x9C\xBC\]/g;
+   $s =~ s/\xC3\xBD/\xC3\[\x9D\xBD\]/g;
+   $s =~ s/\xC3\xBE/\xC3\[\x9E\xBE\]/g;
+   }
+   if ($s =~ /\xC5/) {
+   $s =~ s/\xC5\x91/\xC5\[\x90\x91\]/g;
+   $s =~ s/\xC5\xA1/\xC5\[\xA0\xA1\]/g;
+   $s =~ s/\xC5\xB1/\xC5\[\xB0\xB1\]/g;
+   }
+
+   return $s;
+}
+
+sub extended_lower_case {
+   local($caller, $s) = @_;
+
+   $s =~ tr/A-Z/a-z/;
+
+      # Latin-1
+      if ($s =~ /\xC3[\x80-\x9F]/) {
+         $s =~ s/À/à/g;
+         $s =~ s/Á/á/g;
+         $s =~ s/Â/â/g;
+         $s =~ s/Ã/ã/g;
+         $s =~ s/Ä/ä/g;
+         $s =~ s/Å/å/g;
+         $s =~ s/Æ/æ/g;
+         $s =~ s/Ç/ç/g;
+         $s =~ s/È/è/g;
+         $s =~ s/É/é/g;
+         $s =~ s/Ê/ê/g;
+         $s =~ s/Ë/ë/g;
+         $s =~ s/Ì/ì/g;
+         $s =~ s/Í/í/g;
+         $s =~ s/Î/î/g;
+         $s =~ s/Ï/ï/g;
+         $s =~ s/Ð/ð/g;
+         $s =~ s/Ñ/ñ/g;
+         $s =~ s/Ò/ò/g;
+         $s =~ s/Ó/ó/g;
+         $s =~ s/Ô/ô/g;
+         $s =~ s/Õ/õ/g;
+         $s =~ s/Ö/ö/g;
+         $s =~ s/Ø/ø/g;
+         $s =~ s/Ù/ù/g;
+         $s =~ s/Ú/ú/g;
+         $s =~ s/Û/û/g;
+         $s =~ s/Ü/ü/g;
+         $s =~ s/Ý/ý/g;
+         $s =~ s/Þ/þ/g;
+      }
+      # Latin Extended-A
+      if ($s =~ /[\xC4-\xC5][\x80-\xBF]/) {
+         $s =~ s/Ā/ā/g;
+         $s =~ s/Ă/ă/g;
+         $s =~ s/Ą/ą/g;
+         $s =~ s/Ć/ć/g;
+         $s =~ s/Ĉ/ĉ/g;
+         $s =~ s/Ċ/ċ/g;
+         $s =~ s/Č/č/g;
+         $s =~ s/Ď/ď/g;
+         $s =~ s/Đ/đ/g;
+         $s =~ s/Ē/ē/g;
+         $s =~ s/Ĕ/ĕ/g;
+         $s =~ s/Ė/ė/g;
+         $s =~ s/Ę/ę/g;
+         $s =~ s/Ě/ě/g;
+         $s =~ s/Ĝ/ĝ/g;
+         $s =~ s/Ğ/ğ/g;
+         $s =~ s/Ġ/ġ/g;
+         $s =~ s/Ģ/ģ/g;
+         $s =~ s/Ĥ/ĥ/g;
+         $s =~ s/Ħ/ħ/g;
+         $s =~ s/Ĩ/ĩ/g;
+         $s =~ s/Ī/ī/g;
+         $s =~ s/Ĭ/ĭ/g;
+         $s =~ s/Į/į/g;
+         $s =~ s/İ/ı/g;
+         $s =~ s/IJ/ij/g;
+         $s =~ s/Ĵ/ĵ/g;
+         $s =~ s/Ķ/ķ/g;
+         $s =~ s/Ĺ/ĺ/g;
+         $s =~ s/Ļ/ļ/g;
+         $s =~ s/Ľ/ľ/g;
+         $s =~ s/Ŀ/ŀ/g;
+         $s =~ s/Ł/ł/g;
+         $s =~ s/Ń/ń/g;
+         $s =~ s/Ņ/ņ/g;
+         $s =~ s/Ň/ň/g;
+         $s =~ s/Ŋ/ŋ/g;
+         $s =~ s/Ō/ō/g;
+         $s =~ s/Ŏ/ŏ/g;
+         $s =~ s/Ő/ő/g;
+         $s =~ s/Œ/œ/g;
+         $s =~ s/Ŕ/ŕ/g;
+         $s =~ s/Ŗ/ŗ/g;
+         $s =~ s/Ř/ř/g;
+         $s =~ s/Ś/ś/g;
+         $s =~ s/Ŝ/ŝ/g;
+         $s =~ s/Ş/ş/g;
+         $s =~ s/Š/š/g;
+         $s =~ s/Ţ/ţ/g;
+         $s =~ s/Ť/ť/g;
+         $s =~ s/Ŧ/ŧ/g;
+         $s =~ s/Ũ/ũ/g;
+         $s =~ s/Ū/ū/g;
+         $s =~ s/Ŭ/ŭ/g;
+         $s =~ s/Ů/ů/g;
+         $s =~ s/Ű/ű/g;
+         $s =~ s/Ų/ų/g;
+         $s =~ s/Ŵ/ŵ/g;
+         $s =~ s/Ŷ/ŷ/g;
+         $s =~ s/Ź/ź/g;
+         $s =~ s/Ż/ż/g;
+         $s =~ s/Ž/ž/g;
+      }
+      # Greek letters
+      if ($s =~ /\xCE[\x86-\xAB]/) {
+         $s =~ s/Α/α/g;
+         $s =~ s/Β/β/g;
+         $s =~ s/Γ/γ/g;
+         $s =~ s/Δ/δ/g;
+         $s =~ s/Ε/ε/g;
+         $s =~ s/Ζ/ζ/g;
+         $s =~ s/Η/η/g;
+         $s =~ s/Θ/θ/g;
+         $s =~ s/Ι/ι/g;
+         $s =~ s/Κ/κ/g;
+         $s =~ s/Λ/λ/g;
+         $s =~ s/Μ/μ/g;
+         $s =~ s/Ν/ν/g;
+         $s =~ s/Ξ/ξ/g;
+         $s =~ s/Ο/ο/g;
+         $s =~ s/Π/π/g;
+         $s =~ s/Ρ/ρ/g;
+         $s =~ s/Σ/σ/g;
+         $s =~ s/Τ/τ/g;
+         $s =~ s/Υ/υ/g;
+         $s =~ s/Φ/φ/g;
+         $s =~ s/Χ/χ/g;
+         $s =~ s/Ψ/ψ/g;
+         $s =~ s/Ω/ω/g;
+         $s =~ s/Ϊ/ϊ/g;
+         $s =~ s/Ϋ/ϋ/g;
+         $s =~ s/Ά/ά/g;
+         $s =~ s/Έ/έ/g;
+         $s =~ s/Ή/ή/g;
+         $s =~ s/Ί/ί/g;
+         $s =~ s/Ό/ό/g;
+         $s =~ s/Ύ/ύ/g;
+         $s =~ s/Ώ/ώ/g;
+      }
+      # Cyrillic letters
+      if ($s =~ /\xD0[\x80-\xAF]/) {
+         $s =~ s/А/а/g;
+         $s =~ s/Б/б/g;
+         $s =~ s/В/в/g;
+         $s =~ s/Г/г/g;
+         $s =~ s/Д/д/g;
+         $s =~ s/Е/е/g;
+         $s =~ s/Ж/ж/g;
+         $s =~ s/З/з/g;
+         $s =~ s/И/и/g;
+         $s =~ s/Й/й/g;
+         $s =~ s/К/к/g;
+         $s =~ s/Л/л/g;
+         $s =~ s/М/м/g;
+         $s =~ s/Н/н/g;
+         $s =~ s/О/о/g;
+         $s =~ s/П/п/g;
+         $s =~ s/Р/р/g;
+         $s =~ s/С/с/g;
+         $s =~ s/Т/т/g;
+         $s =~ s/У/у/g;
+         $s =~ s/Ф/ф/g;
+         $s =~ s/Х/х/g;
+         $s =~ s/Ц/ц/g;
+         $s =~ s/Ч/ч/g;
+         $s =~ s/Ш/ш/g;
+         $s =~ s/Щ/щ/g;
+         $s =~ s/Ъ/ъ/g;
+         $s =~ s/Ы/ы/g;
+         $s =~ s/Ь/ь/g;
+         $s =~ s/Э/э/g;
+         $s =~ s/Ю/ю/g;
+         $s =~ s/Я/я/g;
+         $s =~ s/Ѐ/ѐ/g;
+         $s =~ s/Ё/ё/g;
+         $s =~ s/Ђ/ђ/g;
+         $s =~ s/Ѓ/ѓ/g;
+         $s =~ s/Є/є/g;
+         $s =~ s/Ѕ/ѕ/g;
+         $s =~ s/І/і/g;
+         $s =~ s/Ї/ї/g;
+         $s =~ s/Ј/ј/g;
+         $s =~ s/Љ/љ/g;
+         $s =~ s/Њ/њ/g;
+         $s =~ s/Ћ/ћ/g;
+         $s =~ s/Ќ/ќ/g;
+         $s =~ s/Ѝ/ѝ/g;
+         $s =~ s/Ў/ў/g;
+         $s =~ s/Џ/џ/g;
+      }
+      # Fullwidth A-Z
+      if ($s =~ /\xEF\xBC[\xA1-\xBA]/) {
+         $s =~ s/Ａ/ａ/g;
+         $s =~ s/Ｂ/ｂ/g;
+         $s =~ s/Ｃ/ｃ/g;
+         $s =~ s/Ｄ/ｄ/g;
+         $s =~ s/Ｅ/ｅ/g;
+         $s =~ s/Ｆ/ｆ/g;
+         $s =~ s/Ｇ/ｇ/g;
+         $s =~ s/Ｈ/ｈ/g;
+         $s =~ s/Ｉ/ｉ/g;
+         $s =~ s/Ｊ/ｊ/g;
+         $s =~ s/Ｋ/ｋ/g;
+         $s =~ s/Ｌ/ｌ/g;
+         $s =~ s/Ｍ/ｍ/g;
+         $s =~ s/Ｎ/ｎ/g;
+         $s =~ s/Ｏ/ｏ/g;
+         $s =~ s/Ｐ/ｐ/g;
+         $s =~ s/Ｑ/ｑ/g;
+         $s =~ s/Ｒ/ｒ/g;
+         $s =~ s/Ｓ/ｓ/g;
+         $s =~ s/Ｔ/ｔ/g;
+         $s =~ s/Ｕ/ｕ/g;
+         $s =~ s/Ｖ/ｖ/g;
+         $s =~ s/Ｗ/ｗ/g;
+         $s =~ s/Ｘ/ｘ/g;
+         $s =~ s/Ｙ/ｙ/g;
+         $s =~ s/Ｚ/ｚ/g;
+      }
+
+   return $s;
+}
+
+sub extended_upper_case {
+   local($caller, $s) = @_;
+
+   $s =~ tr/a-z/A-Z/;
+   return $s unless $s =~ /[\xC3-\xC5][\x80-\xBF]/;
+
+   $s =~ s/\xC3\xA0/\xC3\x80/g;
+   $s =~ s/\xC3\xA1/\xC3\x81/g;
+   $s =~ s/\xC3\xA2/\xC3\x82/g;
+   $s =~ s/\xC3\xA3/\xC3\x83/g;
+   $s =~ s/\xC3\xA4/\xC3\x84/g;
+   $s =~ s/\xC3\xA5/\xC3\x85/g;
+   $s =~ s/\xC3\xA6/\xC3\x86/g;
+   $s =~ s/\xC3\xA7/\xC3\x87/g;
+   $s =~ s/\xC3\xA8/\xC3\x88/g;
+   $s =~ s/\xC3\xA9/\xC3\x89/g;
+   $s =~ s/\xC3\xAA/\xC3\x8A/g;
+   $s =~ s/\xC3\xAB/\xC3\x8B/g;
+   $s =~ s/\xC3\xAC/\xC3\x8C/g;
+   $s =~ s/\xC3\xAD/\xC3\x8D/g;
+   $s =~ s/\xC3\xAE/\xC3\x8E/g;
+   $s =~ s/\xC3\xAF/\xC3\x8F/g;
+   $s =~ s/\xC3\xB0/\xC3\x90/g;
+   $s =~ s/\xC3\xB1/\xC3\x91/g;
+   $s =~ s/\xC3\xB2/\xC3\x92/g;
+   $s =~ s/\xC3\xB3/\xC3\x93/g;
+   $s =~ s/\xC3\xB4/\xC3\x94/g;
+   $s =~ s/\xC3\xB5/\xC3\x95/g;
+   $s =~ s/\xC3\xB6/\xC3\x96/g;
+   $s =~ s/\xC3\xB8/\xC3\x98/g;
+   $s =~ s/\xC3\xB9/\xC3\x99/g;
+   $s =~ s/\xC3\xBA/\xC3\x9A/g;
+   $s =~ s/\xC3\xBB/\xC3\x9B/g;
+   $s =~ s/\xC3\xBC/\xC3\x9C/g;
+   $s =~ s/\xC3\xBD/\xC3\x9D/g;
+   $s =~ s/\xC3\xBE/\xC3\x9E/g;
+
+   $s =~ s/\xC5\x91/\xC5\x90/g;
+   $s =~ s/\xC5\xA1/\xC5\xA0/g;
+   $s =~ s/\xC5\xB1/\xC5\xB0/g;
+   return $s unless $s =~ /[\xC3-\xC5][\x80-\xBF]/;
+
+   return $s;
+}
+
+sub extended_first_upper_case {
+   local($caller, $s) = @_;
+
+   if (($first_char, $rest) = ($s =~ /^([\x00-\x7F]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF])(.*)$/)) {
+      return $caller->extended_upper_case($first_char) . $rest;
+   } else {
+      return $s;
+   }
+}
+
+sub repair_doubly_converted_utf8_strings {
+   local($caller, $s) = @_;
+
+   if ($s =~ /\xC3[\x82-\x85]\xC2[\x80-\xBF]/) {
+      $s =~ s/\xC3\x82\xC2([\x80-\xBF])/\xC2$1/g;
+      $s =~ s/\xC3\x83\xC2([\x80-\xBF])/\xC3$1/g;
+      $s =~ s/\xC3\x84\xC2([\x80-\xBF])/\xC4$1/g;
+      $s =~ s/\xC3\x85\xC2([\x80-\xBF])/\xC5$1/g;
+   }
+   return $s;
+}
+
+sub repair_misconverted_windows_to_utf8_strings {
+   local($caller, $s) = @_;
+
+   # correcting conversions of UTF8 using Latin1-to-UTF converter
+   if ($s =~ /\xC3\xA2\xC2\x80\xC2[\x90-\xEF]/) {
+      my $result = "";
+      while (($pre,$last_c,$post) = ($s =~ /^(.*?)\xC3\xA2\xC2\x80\xC2([\x90-\xEF])(.*)$/s)) {
+         $result .= "$pre\xE2\x80$last_c";
+         $s = $post;
+      }
+      $result .= $s;
+      $s = $result;
+   }
+   # correcting conversions of Windows1252-to-UTF8 using Latin1-to-UTF converter
+   if ($s =~ /\xC2[\x80-\x9F]/) {
+      my $result = "";
+      while (($pre,$c_windows,$post) = ($s =~ /^(.*?)\xC2([\x80-\x9F])(.*)$/s)) {
+	 $c_utf8 = $caller->windows1252_to_utf8($c_windows, 0);
+         $result .= ($c_utf8 eq "?") ? ($pre . "\xC2" . $c_windows) : "$pre$c_utf8";
+         $s = $post;
+      }
+      $result .= $s;
+      $s = $result;
+   }
+   if ($s =~ /\xC3/) {
+      $s =~ s/\xC3\xA2\xE2\x80\x9A\xC2\xAC/\xE2\x82\xAC/g;     # x80 -> Euro sign
+                                                               # x81 codepoint undefined in Windows 1252
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC5\xA1/\xE2\x80\x9A/g;     # x82 -> single low-9 quotation mark
+      $s =~ s/\xC3\x86\xE2\x80\x99/\xC6\x92/g;                 # x83 -> Latin small letter f with hook
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC5\xBE/\xE2\x80\x9E/g;     # x84 -> double low-9 quotation mark
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA6/\xE2\x80\xA6/g;     # x85 -> horizontal ellipsis
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA0/\xE2\x80\xA0/g;     # x86 -> dagger
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA1/\xE2\x80\xA1/g;     # x87 -> double dagger
+      $s =~ s/\xC3\x8B\xE2\x80\xA0/\xCB\x86/g;                 # x88 -> modifier letter circumflex accent
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xB0/\xE2\x80\xB0/g;     # x89 -> per mille sign
+      $s =~ s/\xC3\x85\xC2\xA0/\xC5\xA0/g;                     # x8A -> Latin capital letter S with caron
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xB9/\xE2\x80\xB9/g;     # x8B -> single left-pointing angle quotation mark
+      $s =~ s/\xC3\x85\xE2\x80\x99/\xC5\x92/g;                 # x8C -> Latin capital ligature OE
+                                                               # x8D codepoint undefined in Windows 1252
+      $s =~ s/\xC3\x85\xC2\xBD/\xC5\xBD/g;                     # x8E -> Latin capital letter Z with caron
+                                                               # x8F codepoint undefined in Windows 1252
+                                                               # x90 codepoint undefined in Windows 1252
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xCB\x9C/\xE2\x80\x98/g;     # x91 a-circumflex+euro+small tilde -> left single quotation mark
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xE2\x84\xA2/\xE2\x80\x99/g; # x92 a-circumflex+euro+trademark -> right single quotation mark
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC5\x93/\xE2\x80\x9C/g;     # x93 a-circumflex+euro+Latin small ligature oe -> left double quotation mark
+							       # x94 maps through undefined intermediate code point
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA2/\xE2\x80\xA2/g;     # x95 a-circumflex+euro+cent sign -> bullet
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xE2\x80\x9C/\xE2\x80\x93/g; # x96 a-circumflex+euro+left double quotation mark -> en dash
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xE2\x80\x9D/\xE2\x80\x94/g; # x97 a-circumflex+euro+right double quotation mark -> em dash
+      $s =~ s/\xC3\x8B\xC5\x93/\xCB\x9C/g;                     # x98 Latin capital e diaeresis+Latin small ligature oe -> small tilde
+      $s =~ s/\xC3\xA2\xE2\x80\x9E\xC2\xA2/\xE2\x84\xA2/g;     # x99 -> trade mark sign
+      $s =~ s/\xC3\x85\xC2\xA1/\xC5\xA1/g;                     # x9A -> Latin small letter s with caron
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xBA/\xE2\x80\xBA/g;     # x9B -> single right-pointing angle quotation mark
+      $s =~ s/\xC3\x85\xE2\x80\x9C/\xC5\x93/g;                 # x9C -> Latin small ligature oe
+							       # x9D codepoint undefined in Windows 1252
+      $s =~ s/\xC3\x85\xC2\xBE/\xC5\xBE/g;                     # x9E -> Latin small letter z with caron
+      $s =~ s/\xC3\x85\xC2\xB8/\xC5\xB8/g;                     # x9F -> Latin capital letter Y with diaeresis 
+      $s =~ s/\xC3\xAF\xC2\xBF\xC2\xBD/\xEF\xBF\xBD/g;         # replacement character
+   }
+
+   return $s;
+}
+
+sub latin1_to_utf {
+   local($caller, $s) = @_;
+
+   my $result = "";
+   while (($pre,$c,$post) = ($s =~ /^(.*?)([\x80-\xFF])(.*)$/s)) {
+      $result .= $pre;
+      if ($c =~ /^[\x80-\xBF]$/) {
+         $result .= "\xC2$c";
+      } elsif ($c =~ /^[\xC0-\xFF]$/) {
+         $c =~ tr/[\xC0-\xFF]/[\x80-\xBF]/;
+         $result .= "\xC3$c";
+      }
+      $s = $post;
+   }
+   $result .= $s;
+   return $result;
+}
+
+sub character_type_is_letter_type {
+   local($caller, $char_type) = @_;
+
+   return ($char_type =~ /\b((CJK|hiragana|kana|katakana)\s+character|diacritic|letter|syllable)\b/);
+}
+
+sub character_type {
+   local($caller, $c) = @_;
+
+   if ($c =~ /^[\x00-\x7F]/) {
+      return "XML tag" if $c =~ /^<.*>$/;
+      return "ASCII Latin letter" if $c =~ /^[a-z]$/i;
+      return "ASCII digit" if $c =~ /^[0-9]$/i;
+      return "ASCII whitespace" if $c =~ /^[\x09-\x0D\x20]$/;
+      return "ASCII control-character" if $c =~ /^[\x00-\x1F\x7F]$/;
+      return "ASCII currency" if $c eq "\$";
+      return "ASCII punctuation";
+   } elsif ($c =~ /^[\xC0-\xDF]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /^[\xC0-\xDF][\x80-\xBF]$/;
+      return "non-shortest-UTF8 (invalid)" if $c =~ /[\xC0-\xC1]/;
+      return "non-ASCII control-character" if $c =~ /\xC2[\x80-\x9F]/;
+      return "non-ASCII whitespace" if $c =~ /\xC2\xA0/;
+      return "non-ASCII currency" if $c =~ /\xC2[\xA2-\xA5]/;
+      return "fraction" if $c =~ /\xC2[\xBC-\xBE]/; # NEW
+      return "superscript digit"  if $c =~ /\xC2[\xB2\xB3\xB9]/;
+      return "non-ASCII Latin letter" if $c =~ /\xC2\xB5/; # micro sign
+      return "non-ASCII punctuation" if $c =~ /\xC2[\xA0-\xBF]/;
+      return "non-ASCII punctuation" if $c =~ /\xC3[\x97\xB7]/;
+      return "non-ASCII Latin letter" if $c =~ /\xC3[\x80-\xBF]/;
+      return "Latin ligature letter" if $c =~ /\xC4[\xB2\xB3]/;
+      return "Latin ligature letter" if $c =~ /\xC5[\x92\x93]/;
+      return "non-ASCII Latin letter" if $c =~ /[\xC4-\xC8]/;
+      return "non-ASCII Latin letter" if $c =~ /\xC9[\x80-\x8F]/;
+      return "IPA" if $c =~ /\xC9[\x90-\xBF]/;
+      return "IPA" if $c =~ /\xCA[\x80-\xBF]/;
+      return "IPA" if $c =~ /\xCB[\x80-\xBF]/;
+      return "combining-diacritic" if $c =~ /\xCC[\x80-\xBF]/;
+      return "combining-diacritic" if $c =~ /\xCD[\x80-\xAF]/;
+      return "Greek punctuation" if $c =~ /\xCD[\xBE]/; # Greek question mark
+      return "Greek punctuation" if $c =~ /\xCE[\x87]/; # Greek semicolon
+      return "Greek letter" if $c =~ /\xCD[\xB0-\xBF]/;
+      return "Greek letter" if $c =~ /\xCE/;
+      return "Greek letter" if $c =~ /\xCF[\x80-\xA1\xB3\xB7\xB8\xBA\xBB]/;
+      return "Coptic letter" if $c =~ /\xCF[\xA2-\xAF]/;
+      return "Cyrillic letter" if $c =~ /[\xD0-\xD3]/;
+      return "Cyrillic letter" if $c =~ /\xD4[\x80-\xAF]/;
+      return "Armenian punctuation" if $c =~ /\xD5[\x9A-\x9F]/;
+      return "Armenian punctuation" if $c =~ /\xD6[\x89-\x8F]/;
+      return "Armenian letter" if $c =~ /\xD4[\xB0-\xBF]/;
+      return "Armenian letter" if $c =~ /\xD5/;
+      return "Armenian letter" if $c =~ /\xD6[\x80-\x8F]/;
+      return "Hebrew accent" if $c =~ /\xD6[\x91-\xAE]/;
+      return "Hebrew punctuation" if $c =~ /\xD6\xBE/;
+      return "Hebrew punctuation" if $c =~ /\xD7[\x80\x83\x86\xB3\xB4]/;
+      return "Hebrew point" if $c =~ /\xD6[\xB0-\xBF]/;
+      return "Hebrew point" if $c =~ /\xD7[\x81\x82\x87]/;
+      return "Hebrew letter" if $c =~ /\xD7[\x90-\xB2]/;
+      return "other Hebrew" if $c =~ /\xD6[\x90-\xBF]/;
+      return "other Hebrew" if $c =~ /\xD7/;
+      return "Arabic currency" if $c =~ /\xD8\x8B/; # Afghani sign
+      return "Arabic punctuation" if $c =~ /\xD8[\x89-\x8D\x9B\x9E\x9F]/;
+      return "Arabic punctuation" if $c =~ /\xD9[\xAA-\xAD]/;
+      return "Arabic punctuation" if $c =~ /\xDB[\x94]/;
+      return "Arabic tatweel" if $c =~ /\xD9\x80/;
+      return "Arabic letter"  if $c =~ /\xD8[\xA0-\xBF]/;
+      return "Arabic letter"  if $c =~ /\xD9[\x81-\x9F]/;
+      return "Arabic letter"  if $c =~ /\xD9[\xAE-\xBF]/;
+      return "Arabic letter"  if $c =~ /\xDA[\x80-\xBF]/;
+      return "Arabic letter"  if $c =~ /\xDB[\x80-\x95]/;
+      return "Arabic Indic digit" if $c =~ /\xD9[\xA0-\xA9]/;
+      return "Arabic Indic digit" if $c =~ /\xDB[\xB0-\xB9]/;
+      return "other Arabic" if $c =~ /[\xD8-\xDB]/;
+      return "Syriac punctuation" if $c =~ /\xDC[\x80-\x8F]/;
+      return "Syriac letter" if $c =~ /\xDC[\x90-\xAF]/;
+      return "Syriac diacritic" if $c =~ /\xDC[\xB0-\xBF]/;
+      return "Syriac diacritic" if $c =~ /\xDD[\x80-\x8A]/;
+      return "Thaana letter" if $c =~ /\xDE/;
+   } elsif ($c =~ /^[\xE0-\xEF]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /^[\xE0-\xEF][\x80-\xBF]{2,2}$/;
+      return "non-shortest-UTF8 (invalid)" if $c =~ /\xE0[\x80-\x9F]/;
+      return "Arabic letter"     if $c =~ /\xE0\xA2[\xA0-\xBF]/; # extended letters
+      return "other Arabic"      if $c =~ /\xE0\xA3/; # extended characters
+      return "Devanagari punctuation" if $c =~ /\xE0\xA5[\xA4\xA5]/; # danda, double danda
+      return "Devanagari digit" if $c =~ /\xE0\xA5[\xA6-\xAF]/;
+      return "Devanagari letter" if $c =~ /\xE0[\xA4-\xA5]/;
+      return "Bengali digit" if $c =~ /\xE0\xA7[\xA6-\xAF]/;
+      return "Bengali currency" if $c =~ /\xE0\xA7[\xB2-\xB9]/;
+      return "Bengali letter" if $c =~ /\xE0[\xA6-\xA7]/;
+      return "Gurmukhi digit" if $c =~ /\xE0\xA9[\xA6-\xAF]/;
+      return "Gurmukhi letter" if $c =~ /\xE0[\xA8-\xA9]/;
+      return "Gujarati digit" if $c =~ /\xE0\xAB[\xA6-\xAF]/;
+      return "Gujarati letter" if $c =~ /\xE0[\xAA-\xAB]/;
+      return "Oriya digit" if $c =~ /\xE0\xAD[\xA6-\xAF]/;
+      return "Oriya fraction" if $c =~ /\xE0\xAD[\xB2-\xB7]/;
+      return "Oriya letter" if $c =~ /\xE0[\xAC-\xAD]/;
+      return "Tamil digit" if $c =~ /\xE0\xAF[\xA6-\xAF]/;
+      return "Tamil number" if $c =~ /\xE0\xAF[\xB0-\xB2]/; # number (10, 100, 1000)
+      return "Tamil letter" if $c =~ /\xE0[\xAE-\xAF]/;
+      return "Telegu digit" if $c =~ /\xE0\xB1[\xA6-\xAF]/;
+      return "Telegu fraction" if $c =~ /\xE0\xB1[\xB8-\xBE]/;
+      return "Telegu letter" if $c =~ /\xE0[\xB0-\xB1]/;
+      return "Kannada digit" if $c =~ /\xE0\xB3[\xA6-\xAF]/;
+      return "Kannada letter" if $c =~ /\xE0[\xB2-\xB3]/;
+      return "Malayalam digit" if $c =~ /\xE0\xB5[\x98-\x9E\xA6-\xB8]/;
+      return "Malayalam punctuation" if $c =~ /\xE0\xB5\xB9/; # date mark
+      return "Malayalam letter" if $c =~ /\xE0[\xB4-\xB5]/;
+      return "Sinhala digit" if $c =~ /\xE0\xB7[\xA6-\xAF]/;
+      return "Sinhala punctuation" if $c =~ /\xE0\xB7\xB4/;
+      return "Sinhala letter" if $c =~ /\xE0[\xB6-\xB7]/;
+      return "Thai currency" if $c =~ /\xE0\xB8\xBF/;
+      return "Thai digit" if $c =~ /\xE0\xB9[\x90-\x99]/;
+      return "Thai character" if $c =~ /\xE0[\xB8-\xB9]/;
+      return "Lao punctuation" if $c =~ /\xE0\xBA\xAF/; # Lao ellipsis
+      return "Lao digit" if $c =~ /\xE0\xBB[\x90-\x99]/;
+      return "Lao character" if $c =~ /\xE0[\xBA-\xBB]/;
+      return "Tibetan punctuation" if $c =~ /\xE0\xBC[\x81-\x94]/;
+      return "Tibetan sign"        if $c =~ /\xE0\xBC[\x95-\x9F]/;
+      return "Tibetan digit"       if $c =~ /\xE0\xBC[\xA0-\xB3]/;
+      return "Tibetan punctuation" if $c =~ /\xE0\xBC[\xB4-\xBD]/;
+      return "Tibetan letter" if $c =~ /\xE0[\xBC-\xBF]/;
+      return "Myanmar digit" if $c =~ /\xE1\x81[\x80-\x89]/;
+      return "Myanmar digit" if $c =~ /\xE1\x82[\x90-\x99]/; # Myanmar Shan digits
+      return "Myanmar punctuation" if $c =~ /\xE1\x81[\x8A-\x8B]/;
+      return "Myanmar letter" if $c =~ /\xE1[\x80-\x81]/;
+      return "Myanmar letter" if $c =~ /\xE1\x82[\x80-\x9F]/;
+      return "Georgian punctuation" if $c =~ /\xE1\x83\xBB/;
+      return "Georgian letter" if $c =~ /\xE1\x82[\xA0-\xBF]/;
+      return "Georgian letter" if $c =~ /\xE1\x83/;
+      return "Georgian letter" if $c =~ /\xE1\xB2[\x90-\xBF]/; # Georgian Mtavruli capital letters
+      return "Georgian letter" if $c =~ /\xE2\xB4[\x80-\xAF]/; # Georgian small letters (Khutsuri)
+      return "Korean Hangul letter" if $c =~ /\xE1[\x84-\x87]/;
+      return "Ethiopic punctuation" if $c =~ /\xE1\x8D[\xA0-\xA8]/;
+      return "Ethiopic digit" if $c =~ /\xE1\x8D[\xA9-\xB1]/;
+      return "Ethiopic number" if $c =~ /\xE1\x8D[\xB2-\xBC]/;
+      return "Ethiopic syllable" if $c =~ /\xE1[\x88-\x8D]/;
+      return "Cherokee letter" if $c =~ /\xE1\x8E[\xA0-\xBF]/;
+      return "Cherokee letter" if $c =~ /\xE1\x8F/;
+      return "Canadian punctuation" if $c =~ /\xE1\x90\x80/; # Canadian Syllabics hyphen
+      return "Canadian punctuation" if $c =~ /\xE1\x99\xAE/; # Canadian Syllabics full stop
+      return "Canadian syllable" if $c =~ /\xE1[\x90-\x99]/;
+      return "Canadian syllable" if $c =~ /\xE1\xA2[\xB0-\xBF]/;
+      return "Canadian syllable" if $c =~ /\xE1\xA3/;
+      return "Ogham whitespace" if $c =~ /\xE1\x9A\x80/;
+      return "Ogham letter" if $c =~ /\xE1\x9A[\x81-\x9A]/;
+      return "Ogham punctuation" if $c =~ /\xE1\x9A[\x9B-\x9C]/;
+      return "Runic punctuation" if $c =~ /\xE1\x9B[\xAB-\xAD]/;
+      return "Runic letter" if $c =~ /\xE1\x9A[\xA0-\xBF]/;
+      return "Runic letter" if $c =~ /\xE1\x9B/;
+      return "Khmer currency" if $c =~ /\xE1\x9F\x9B/;
+      return "Khmer digit" if $c =~ /\xE1\x9F[\xA0-\xA9]/;
+      return "Khmer letter" if $c =~ /\xE1[\x9E-\x9F]/;
+      return "Mongolian punctuation" if $c =~ /\xE1\xA0[\x80-\x8A]/;
+      return "Mongolian digit"       if $c =~ /\xE1\xA0[\x90-\x99]/;
+      return "Mongolian letter" if $c =~ /\xE1[\xA0-\xA1]/;
+      return "Mongolian letter" if $c =~ /\xE1\xA2[\x80-\xAF]/;
+      return "Buginese letter" if $c =~ /\xE1\xA8[\x80-\x9B]/;
+      return "Buginese punctuation" if $c =~ /\xE1\xA8[\x9E-\x9F]/;
+      return "Balinese letter" if $c =~ /\xE1\xAC/;
+      return "Balinese letter" if $c =~ /\xE1\xAD[\x80-\x8F]/;
+      return "Balinese digit" if $c =~ /\xE1\xAD[\x90-\x99]/;
+      return "Balinese puncutation" if $c =~ /\xE1\xAD[\x9A-\xA0]/;
+      return "Balinese symbol" if $c =~ /\xE1\xAD[\xA1-\xBF]/;
+      return "Sundanese digit" if $c =~ /\xE1\xAE[\xB0-\xB9]/;
+      return "Sundanese letter" if $c =~ /\xE1\xAE/;
+      return "Cyrillic letter" if $c =~ /\xE1\xB2[\x80-\x8F]/;
+      return "Sundanese punctuation" if $c =~ /\xE1\xB3[\x80-\x8F]/;
+      return "IPA" if $c =~ /\xE1[\xB4-\xB6]/;
+      return "non-ASCII Latin letter" if $c =~ /\xE1[\xB8-\xBB]/;
+      return "Greek letter" if $c =~ /\xE1[\xBC-\xBF]/;
+      return "non-ASCII whitespace"  if $c =~ /\xE2\x80[\x80-\x8A\xAF]/;
+      return "zero-width space"      if $c =~ /\xE2\x80\x8B/;
+      return "zero-width non-space"  if $c =~ /\xE2\x80\x8C/;
+      return "zero-width joiner"     if $c =~ /\xE2\x80\x8D/;
+      return "directional mark"      if $c =~ /\xE2\x80[\x8E-\x8F\xAA-\xAE]/;
+      return "non-ASCII punctuation" if $c =~ /\xE2\x80[\x90-\xBF]/;
+      return "non-ASCII punctuation" if $c =~ /\xE2\x81[\x80-\x9E]/;
+      return "superscript letter"    if $c =~ /\xE2\x81[\xB1\xBF]/;
+      return "superscript digit"     if $c =~ /\xE2\x81[\xB0-\xB9]/;
+      return "superscript punctuation" if $c =~ /\xE2\x81[\xBA-\xBE]/;
+      return "subscript digit"       if $c =~ /\xE2\x82[\x80-\x89]/;
+      return "subscript punctuation" if $c =~ /\xE2\x82[\x8A-\x8E]/;
+      return "non-ASCII currency"    if $c =~ /\xE2\x82[\xA0-\xBF]/;
+      return "letterlike symbol"     if $c =~ /\xE2\x84/;
+      return "letterlike symbol"     if $c =~ /\xE2\x85[\x80-\x8F]/;
+      return "fraction"              if $c =~ /\xE2\x85[\x90-\x9E]/; # NEW
+      return "Roman number"          if $c =~ /\xE2\x85[\xA0-\xBF]/; # NEW
+      return "arrow symbol"          if $c =~ /\xE2\x86[\x90-\xBF]/;
+      return "arrow symbol"          if $c =~ /\xE2\x87/;
+      return "mathematical operator" if $c =~ /\xE2[\x88-\x8B]/;
+      return "technical symbol"      if $c =~ /\xE2[\x8C-\x8F]/;
+      return "enclosed alphanumeric" if $c =~ /\xE2\x91[\xA0-\xBF]/;
+      return "enclosed alphanumeric" if $c =~ /\xE2[\x92-\x93]/;
+      return "box drawing" if $c =~ /\xE2[\x94-\x95]/;
+      return "geometric shape" if $c =~ /\xE2\x96[\xA0-\xBF]/;
+      return "geometric shape" if $c =~ /\xE2\x97/;
+      return "pictograph" if $c =~ /\xE2[\x98-\x9E]/;
+      return "arrow symbol"          if $c =~ /\xE2\xAC[\x80-\x91\xB0-\xBF]/;
+      return "geometric shape"       if $c =~ /\xE2\xAC[\x92-\xAF]/;         
+      return "arrow symbol"          if $c =~ /\xE2\xAD[\x80-\x8F\x9A-\xBF]/;
+      return "geometric shape"       if $c =~ /\xE2\xAD[\x90-\x99]/;        
+      return "arrow symbol"          if $c =~ /\xE2\xAE[\x80-\xB9]/;       
+      return "geometric shape"       if $c =~ /\xE2\xAE[\xBA-\xBF]/;      
+      return "geometric shape"       if $c =~ /\xE2\xAF[\x80-\x88\x8A-\x8F]/;
+      return "symbol"                if $c =~ /\xE2[\xAC-\xAF]/;
+      return "Coptic fraction" if $c =~ /\xE2\xB3\xBD/;
+      return "Coptic punctuation" if $c =~ /\xE2\xB3[\xB9-\xBF]/;
+      return "Coptic letter" if $c =~ /\xE2[\xB2-\xB3]/;
+      return "Georgian letter" if $c =~ /\xE2\xB4[\x80-\xAF]/;
+      return "Tifinagh punctuation" if $c =~ /\xE2\xB5\xB0/;
+      return "Tifinagh letter" if $c =~ /\xE2\xB4[\xB0-\xBF]/;
+      return "Tifinagh letter" if $c =~ /\xE2\xB5/;
+      return "Ethiopic syllable" if $c =~ /\xE2\xB6/;
+      return "Ethiopic syllable" if $c =~ /\xE2\xB7[\x80-\x9F]/;
+      return "non-ASCII punctuation" if $c =~ /\xE3\x80[\x80-\x91\x94-\x9F\xB0\xBB-\xBD]/;
+      return "symbol" if $c =~ /\xE3\x80[\x91\x92\xA0\xB6\xB7]/;
+      return "Japanese hiragana character" if $c =~ /\xE3\x81/;
+      return "Japanese hiragana character" if $c =~ /\xE3\x82[\x80-\x9F]/;
+      return "Japanese katakana character" if $c =~ /\xE3\x82[\xA0-\xBF]/;
+      return "Japanese katakana character" if $c =~ /\xE3\x83/;
+      return "Bopomofo letter" if $c =~ /\xE3\x84[\x80-\xAF]/;
+      return "Korean Hangul letter" if $c =~ /\xE3\x84[\xB0-\xBF]/;
+      return "Korean Hangul letter" if $c =~ /\xE3\x85/;
+      return "Korean Hangul letter" if $c =~ /\xE3\x86[\x80-\x8F]/;
+      return "Bopomofo letter" if $c =~ /\xE3\x86[\xA0-\xBF]/;
+      return "CJK stroke" if $c =~ /\xE3\x87[\x80-\xAF]/;
+      return "Japanese kana character" if $c =~ /\xE3\x87[\xB0-\xBF]/;
+      return "CJK symbol" if $c =~ /\xE3[\x88-\x8B]/;
+      return "CJK square Latin abbreviation" if $c =~ /\xE3\x8D[\xB1-\xBA]/;
+      return "CJK square Latin abbreviation" if $c =~ /\xE3\x8E/;
+      return "CJK square Latin abbreviation" if $c =~ /\xE3\x8F[\x80-\x9F\xBF]/;
+      return "CJK character" if $c =~ /\xE4[\xB8-\xBF]/;
+      return "CJK character" if $c =~ /[\xE5-\xE9]/;
+      return "Yi syllable" if $c =~ /\xEA[\x80-\x92]/;
+      return "Lisu letter"      if $c =~ /\xEA\x93[\x90-\xBD]/;
+      return "Lisu punctuation" if $c =~ /\xEA\x93[\xBE-\xBF]/;
+      return "Cyrillic letter" if $c =~ /\xEA\x99/;
+      return "Cyrillic letter" if $c =~ /\xEA\x9A[\x80-\x9F]/;
+      return "modifier tone" if $c =~ /\xEA\x9C[\x80-\xA1]/;
+      return "Javanese punctuation" if $c =~ /\xEA\xA7[\x81-\x8D\x9E-\x9F]/;
+      return "Javanese digit" if $c =~ /\xEA\xA7[\x90-\x99]/;
+      return "Javanese letter" if $c =~ /\xEA\xA6/;
+      return "Javanese letter" if $c =~ /\xEA\xA7[\x80-\x9F]/;
+      return "Ethiopic syllable" if $c =~ /\xEA\xAC[\x80-\xAF]/;
+      return "Cherokee letter" if $c =~ /\xEA\xAD[\xB0-\xBF]/;
+      return "Cherokee letter" if $c =~ /\xEA\xAE/;
+      return "Meetai Mayek digit" if $c =~ /\xEA\xAF[\xB0-\xB9]/;
+      return "Meetai Mayek letter" if $c =~ /\xEA\xAF/;
+      return "Korean Hangul syllable" if $c =~ /\xEA[\xB0-\xBF]/;
+      return "Korean Hangul syllable" if $c =~ /[\xEB-\xEC]/;
+      return "Korean Hangul syllable" if $c =~ /\xED[\x80-\x9E]/;
+      return "Klingon letter" if $c =~ /\xEF\xA3[\x90-\xA9]/;
+      return "Klingon digit" if $c =~ /\xEF\xA3[\xB0-\xB9]/;
+      return "Klingon punctuation" if $c =~ /\xEF\xA3[\xBD-\xBE]/;
+      return "Klingon symbol" if $c =~ /\xEF\xA3\xBF/;
+      return "private use character" if $c =~ /\xEE/;
+      return "Latin typographic ligature" if $c =~ /\xEF\xAC[\x80-\x86]/;
+      return "Hebrew presentation letter" if $c =~ /\xEF\xAC[\x9D-\xBF]/;
+      return "Hebrew presentation letter" if $c =~ /\xEF\xAD[\x80-\x8F]/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xAD[\x90-\xBF]/;
+      return "Arabic presentation letter" if $c =~ /\xEF[\xAE-\xB7]/;
+      return "non-ASCII punctuation" if $c =~ /\xEF\xB8[\x90-\x99]/;
+      return "non-ASCII punctuation" if $c =~ /\xEF\xB8[\xB0-\xBF]/;
+      return "non-ASCII punctuation" if $c =~ /\xEF\xB9[\x80-\xAB]/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xB9[\xB0-\xBF]/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xBA/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xBB[\x80-\xBC]/;
+      return "byte-order mark/zero-width no-break space" if $c eq "\xEF\xBB\xBF";
+      return "fullwidth currency" if $c =~ /\xEF\xBC\x84/;
+      return "fullwidth digit" if $c =~ /\xEF\xBC[\x90-\x99]/;
+      return "fullwidth Latin letter" if $c =~ /\xEF\xBC[\xA1-\xBA]/;
+      return "fullwidth Latin letter" if $c =~ /\xEF\xBD[\x81-\x9A]/;
+      return "fullwidth punctuation" if $c =~ /\xEF\xBC/;
+      return "fullwidth punctuation" if $c =~ /\xEF\xBD[\x9B-\xA4]/;
+      return "halfwidth Japanese punctuation" if $c =~ /\xEF\xBD[\xA1-\xA4]/;
+      return "halfwidth Japanese katakana character" if $c =~ /\xEF\xBD[\xA5-\xBF]/;
+      return "halfwidth Japanese katakana character" if $c =~ /\xEF\xBE[\x80-\x9F]/;
+      return "fullwidth currency" if $c =~ /\xEF\xBF[\xA0-\xA6]/;
+      return "replacement character" if $c eq "\xEF\xBF\xBD";
+   } elsif ($c =~ /[\xF0-\xF7]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /[\xF0-\xF7][\x80-\xBF]{3,3}$/;
+      return "non-shortest-UTF8 (invalid)" if $c =~ /\xF0[\x80-\x8F]/;
+      return "Linear B syllable" if $c =~ /\xF0\x90\x80/;
+      return "Linear B syllable" if $c =~ /\xF0\x90\x81[\x80-\x8F]/;
+      return "Linear B symbol"   if $c =~ /\xF0\x90\x81[\x90-\x9F]/;
+      return "Linear B ideogram" if $c =~ /\xF0\x90[\x82-\x83]/;
+      return "Gothic letter" if $c =~ /\xF0\x90\x8C[\xB0-\xBF]/;
+      return "Gothic letter" if $c =~ /\xF0\x90\x8D[\x80-\x8F]/;
+      return "Phoenician letter" if $c =~ /\xF0\x90\xA4[\x80-\x95]/;
+      return "Phoenician number" if $c =~ /\xF0\x90\xA4[\x96-\x9B]/;
+      return "Phoenician punctuation" if $c =~ /\xF0\x90\xA4\x9F/; # word separator
+      return "Old Hungarian number" if $c =~ /\xF0\x90\xB3[\xBA-\xBF]/;
+      return "Old Hungarian letter" if $c =~ /\xF0\x90[\xB2-\xB3]/;
+      return "Cuneiform digit" if $c =~ /\xF0\x92\x90/; # numberic sign
+      return "Cuneiform digit" if $c =~ /\xF0\x92\x91[\x80-\xAF]/; # numberic sign
+      return "Cuneiform punctuation" if $c =~ /\xF0\x92\x91[\xB0-\xBF]/;
+      return "Cuneiform sign" if $c =~ /\xF0\x92[\x80-\x95]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x81\xA8/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x82[\xAD-\xB6]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x86[\x90\xBC-\xBF]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x87[\x80-\x84]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x8D[\xA2-\xAB]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x8E[\x86-\x92]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x8F[\xBA-\xBF]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x90[\x80-\x83]/;
+      return "Egyptian hieroglyph" if $c =~ /\xF0\x93[\x80-\x90]/;
+      return "enclosed alphanumeric" if $c =~ /\xF0\x9F[\x84-\x87]/;
+      return "Mahjong symbol" if $c =~ /\xF0\x9F\x80[\x80-\xAF]/;
+      return "Domino symbol" if $c =~ /\xF0\x9F\x80[\xB0-\xBF]/;
+      return "Domino symbol" if $c =~ /\xF0\x9F\x81/;
+      return "Domino symbol" if $c =~ /\xF0\x9F\x82[\x80-\x9F]/;
+      return "Playing card symbol" if $c =~ /\xF0\x9F\x82[\xA0-\xBF]/;
+      return "Playing card symbol" if $c =~ /\xF0\x9F\x83/;
+      return "CJK symbol" if $c =~ /\xF0\x9F[\x88-\x8B]/;
+      return "pictograph" if $c =~ /\xF0\x9F[\x8C-\x9B]/;
+      return "geometric shape" if $c =~ /\xF0\x9F[\x9E-\x9F]/;
+      return "non-ASCII punctuation" if $c =~ /\xF0\x9F[\xA0-\xA3]/;
+      return "pictograph" if $c =~ /\xF0\x9F[\xA4-\xAB]/;
+      return "CJK character" if $c =~ /\xF0[\xA0-\xAF]/;
+      return "tag" if $c =~ /\xF3\xA0[\x80-\x81]/;
+      return "variation selector" if $c =~ /\xF3\xA0[\x84-\x87]/;
+      return "private use character" if $c =~ /\xF3[\xB0-\xBF]/;
+      return "private use character" if $c =~ /\xF4[\x80-\x8F]/;
+      # ...
+   } elsif ($c =~ /[\xF8-\xFB]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /[\xF8-\xFB][\x80-\xBF]{4,4}$/;
+   } elsif ($c =~ /[\xFC-\xFD]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /[\xFC-\xFD][\x80-\xBF]{5,5}$/;
+   } elsif ($c =~ /\xFE/) {
+      return "non-UTF8 (invalid)" unless $c =~ /\xFE][\x80-\xBF]{6,6}$/;
+   } else {
+      return "non-UTF8 (invalid)";
+   }
+   return "other character";
+}
+
+1;
+
+

uroman/lib/NLP/stringDistance.pm

@@ -0,0 +1,724 @@
+################################################################
+#                                                              #
+# stringDistance                                               #
+#                                                              #
+################################################################
+
+package NLP::stringDistance;
+
+use List::Util qw(min max);
+$utf8 = NLP::UTF8;
+$util = NLP::utilities;
+$romanizer = NLP::Romanizer;
+
+%dummy_ht = ();
+
+sub rule_string_expansion {
+   local($this, *ht, $s, $lang_code) = @_;
+   
+   my @characters = $utf8->split_into_utf8_characters($s, "return only chars, return trailing whitespaces", *dummy_ht);
+   foreach $sub_len ((0 .. ($#characters-1))) {
+      my $sub = join("", @characters[0 .. $sub_len]);
+      foreach $super_len ((($sub_len + 1) .. $#characters)) {
+         my $super = join("", @characters[0 .. $super_len]);
+	 # print STDERR "  $sub -> $super\n" unless $ht{RULE_STRING_EXPANSION}->{$lang_code}->{$sub}->{$super};
+	 $ht{RULE_STRING_EXPANSION}->{$lang_code}->{$sub}->{$super} = 1;
+	 $ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$sub} = 1;
+	 # print STDERR "  RULE_STRING_HAS_EXPANSION $lang_code $sub\n";
+      }
+   }
+}
+
+sub load_string_distance_data {
+   local($this, $filename, *ht, $verbose) = @_;
+
+   $verbose = 0 unless defined($verbose);
+   open(IN,$filename) || die "Could not open $filename";
+   my $line_number = 0;
+   my $n_cost_rules = 0;
+   while (<IN>) {
+      $line_number++;
+      my $line = $_;
+      $line =~ s/^\xEF\xBB\xBF//;
+      $line =~ s/\s*$//;
+      next if $line =~ /^\s*(\#.*)?$/;
+      print STDERR "** Warning: line $line_number contains suspicious control character: $line\n" if $line =~ /[\x00-\x1F]/;
+      my $s1 = $util->slot_value_in_double_colon_del_list($line, "s1");
+      my $s2 = $util->slot_value_in_double_colon_del_list($line, "s2");
+      $s1 = $util->dequote_string($s1); # 'can\'t' => can't
+      $s2 = $util->dequote_string($s2);
+      my $cost = $util->slot_value_in_double_colon_del_list($line, "cost");
+      if (($s1 eq "") && ($s2 eq "")) {
+	 print STDERR "Ignoring bad line $line_number in $filename, because both s1 and s2 are empty strings\n";
+         next;
+      }
+      unless ($cost =~ /^\d+(\.\d+)?$/) {
+	 if ($cost eq "") {
+	    print STDERR "Ignoring bad line $line_number in $filename, because of missing cost\n";
+	 } else {
+	    print STDERR "Ignoring bad line $line_number in $filename, because of ill-formed cost $cost\n";
+	 }
+         next;
+      }
+      my $lang_code1_s = $util->slot_value_in_double_colon_del_list($line, "lc1");
+      my $lang_code2_s = $util->slot_value_in_double_colon_del_list($line, "lc2");
+      my @lang_codes_1 = ($lang_code1_s eq "") ? ("") : split(/,\s*/, $lang_code1_s);
+      my @lang_codes_2 = ($lang_code2_s eq "") ? ("") : split(/,\s*/, $lang_code2_s);
+      my $left_context1 = $util->slot_value_in_double_colon_del_list($line, "left1");
+      my $left_context2 = $util->slot_value_in_double_colon_del_list($line, "left2");
+      my $right_context1 = $util->slot_value_in_double_colon_del_list($line, "right1");
+      my $right_context2 = $util->slot_value_in_double_colon_del_list($line, "right2");
+      my $bad_left = $util->slot_value_in_double_colon_del_list($line, "left");
+      if ($bad_left) {
+	 print STDERR "** Warning: slot '::left $bad_left' in line $line_number\n";
+         next;
+      }
+      my $bad_right = $util->slot_value_in_double_colon_del_list($line, "right");
+      if ($bad_right) {
+	 print STDERR "** Warning: slot '::right $bad_right' in line $line_number\n";
+         next;
+      }
+      my $in_lang_codes1 = $util->slot_value_in_double_colon_del_list($line, "in-lc1");
+      my $in_lang_codes2 = $util->slot_value_in_double_colon_del_list($line, "in-lc2");
+      my $out_lang_codes1 = $util->slot_value_in_double_colon_del_list($line, "out-lc1");
+      my $out_lang_codes2 = $util->slot_value_in_double_colon_del_list($line, "out-lc2");
+      if ($left_context1) {
+         if ($left_context1 =~ /^\/.*\/$/) {
+            $left_context1 =~ s/^\///;
+            $left_context1 =~ s/\/$//;
+         } else {
+            print STDERR "Ignoring unrecognized non-regular-express ::left1 $left_context1 in $line_number of $filename\n";
+            $left_context1 = "";
+         }
+      }
+      if ($left_context2) {
+         if ($left_context2 =~ /^\/.*\/$/) {
+            $left_context2 =~ s/^\///;
+            $left_context2 =~ s/\/$//;
+         } else {
+            $left_context2 = "";
+            print STDERR "Ignoring unrecognized non-regular-express ::left2 $left_context2 in $line_number of $filename\n";
+         }
+      }
+      if ($right_context1) {
+         unless ($right_context1 =~ /^(\[[^\[\]]*\])+$/) {
+	    $right_context1 = "";
+	    print STDERR "Ignoring unrecognized right-context ::right1 $right_context1 in $line_number of $filename\n";
+	 }
+      }
+      if ($right_context2) {
+         unless ($right_context2 =~ /^(\[[^\[\]]*\])+$/) {
+	    $right_context2 = "";
+	    print STDERR "Ignoring unrecognized right-context ::right2 $right_context2 in $line_number of $filename\n";
+	 }
+      }
+      foreach $lang_code1 (@lang_codes_1) {
+         foreach $lang_code2 (@lang_codes_2) {
+            $n_cost_rules++;
+            my $cost_rule_id = $n_cost_rules;
+            $ht{COST}->{$lang_code1}->{$lang_code2}->{$s1}->{$s2}->{$cost_rule_id} = $cost;
+            $ht{RULE_STRING}->{$lang_code1}->{$s1} = 1;
+            $ht{RULE_STRING}->{$lang_code2}->{$s2} = 1;
+            $ht{LEFT1}->{$cost_rule_id} = $left_context1;
+            $ht{LEFT2}->{$cost_rule_id} = $left_context2;
+            $ht{RIGHT1}->{$cost_rule_id} = $right_context1;
+            $ht{RIGHT2}->{$cost_rule_id} = $right_context2;
+            $ht{INLC1}->{$cost_rule_id} = $in_lang_codes1;
+            $ht{INLC2}->{$cost_rule_id} = $in_lang_codes2;
+            $ht{OUTLC1}->{$cost_rule_id} = $out_lang_codes1;
+            $ht{OUTLC2}->{$cost_rule_id} = $out_lang_codes2;
+            unless (($s1 eq $s2)
+	         && ($lang_code1 eq $lang_code2)
+	         && ($left_context1 eq $left_context2)
+	         && ($right_context1 eq $right_context2)
+	         && ($in_lang_codes1 eq $in_lang_codes2)
+	         && ($out_lang_codes1 eq $out_lang_codes2)) {
+               $n_cost_rules++;
+               $cost_rule_id = $n_cost_rules;
+               $ht{COST}->{$lang_code2}->{$lang_code1}->{$s2}->{$s1}->{$cost_rule_id} = $cost;
+               $ht{LEFT1}->{$cost_rule_id} = $left_context2;
+               $ht{LEFT2}->{$cost_rule_id} = $left_context1;
+               $ht{RIGHT1}->{$cost_rule_id} = $right_context2;
+               $ht{RIGHT2}->{$cost_rule_id} = $right_context1;
+               $ht{INLC1}->{$cost_rule_id} = $in_lang_codes2;
+               $ht{INLC2}->{$cost_rule_id} = $in_lang_codes1;
+               $ht{OUTLC1}->{$cost_rule_id} = $out_lang_codes2;
+               $ht{OUTLC2}->{$cost_rule_id} = $out_lang_codes1;
+	       # print STDERR "  Flip rule in line $line: $line\n";
+            }
+            $this->rule_string_expansion(*ht, $s1, $lang_code1);
+            $this->rule_string_expansion(*ht, $s2, $lang_code2);
+	 }
+      }
+   }
+   close(IN);
+   print STDERR "Read in $n_cost_rules rules from $line_number lines in $filename\n" if $verbose;
+}
+
+sub romanized_string_to_simple_chart {
+   local($this, $s, *chart_ht) = @_;
+
+   my @characters = $utf8->split_into_utf8_characters($s, "return only chars, return trailing whitespaces", *dummy_ht);
+   $chart_ht{N_CHARS} = $#characters + 1;
+   $chart_ht{N_NODES} = 0;
+   foreach $i ((0 .. $#characters)) {
+      $romanizer->add_node($characters[$i], $i, ($i+1), *chart_ht, "", "");
+   }
+}
+
+sub linearize_chart_points {
+   local($this, *chart_ht, $chart_id, *sd_ht, $verbose) = @_;
+
+   $verbose = 0 unless defined($verbose);
+   print STDERR "Linearize $chart_id\n" if $verbose;
+   my $current_chart_pos = 0;
+   my $current_linear_chart_pos = 0;
+   $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos} = $current_linear_chart_pos;
+   $sd_ht{LINPOS2POS}->{$chart_id}->{$current_linear_chart_pos} = $current_chart_pos;
+   print STDERR "  LINPOS2POS.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos\n" if $verbose;
+   my @end_chart_positions = keys %{$chart_ht{NODES_ENDING_AT}};
+   my $end_chart_pos = (@end_chart_positions) ? max(@end_chart_positions) : 0;
+   $sd_ht{MAXPOS}->{$chart_id} = $end_chart_pos;
+   print STDERR "  Chart span: $current_chart_pos-$end_chart_pos\n" if $verbose;
+   while ($current_chart_pos < $end_chart_pos) {
+      my @node_ids = keys %{$chart_ht{NODES_STARTING_AT}->{$current_chart_pos}};
+      foreach $node_id (@node_ids) {
+	 my $roman_s = $chart_ht{NODE_ROMAN}->{$node_id};
+         my @roman_chars = $utf8->split_into_utf8_characters($roman_s, "return only chars, return trailing whitespaces", *dummy_ht);
+         print STDERR "  $current_chart_pos/$current_linear_chart_pos node: $node_id $roman_s (@roman_chars)\n" if $verbose;
+	 if ($#roman_chars >= 1) {
+	    foreach $i ((1 .. $#roman_chars)) {
+	       $current_linear_chart_pos++;
+	       $sd_ht{SPLITPOS2LINPOS}->{$chart_id}->{$current_chart_pos}->{$node_id}->{$i} = $current_linear_chart_pos;
+	       $sd_ht{LINPOS2SPLITPOS}->{$chart_id}->{$current_linear_chart_pos}->{$current_chart_pos}->{$node_id}->{$i} = 1;
+	       print STDERR "  LINPOS2SPLITPOS.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos NODE: $node_id I: $i\n" if $verbose;
+	    }
+	 }
+      }
+      $current_chart_pos++;
+      if ($util->member($current_chart_pos, @end_chart_positions)) {
+         $current_linear_chart_pos++;
+         $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos} = $current_linear_chart_pos;
+         $sd_ht{LINPOS2POS}->{$chart_id}->{$current_linear_chart_pos} = $current_chart_pos;
+         print STDERR "  LINPOS2POS.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos\n" if $verbose;
+      }
+   }
+   $current_chart_pos = 0;
+   while ($current_chart_pos <= $end_chart_pos) {
+      my $current_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos};
+      $current_linear_chart_pos = "?" unless defined($current_linear_chart_pos);
+      my @node_ids = keys %{$chart_ht{NODES_STARTING_AT}->{$current_chart_pos}};
+      # print STDERR "  LINROM.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos NODES: @node_ids\n" if $verbose;
+      foreach $node_id (@node_ids) {
+	 my $end_pos = $chart_ht{NODE_END}->{$node_id};
+	 my $end_linpos = $sd_ht{POS2LINPOS}->{$chart_id}->{$end_pos};
+	 my $roman_s = $chart_ht{NODE_ROMAN}->{$node_id};
+         my @roman_chars = $utf8->split_into_utf8_characters($roman_s, "return only chars, return trailing whitespaces", *dummy_ht);
+         print STDERR "  LINROM.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos NODE: $node_id CHARS: @roman_chars\n" if $verbose;
+	 if (@roman_chars) {
+            foreach $i ((0 .. $#roman_chars)) {
+	       my $from_linear_chart_pos 
+		  = (($i == 0) 
+		     ? $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos}
+	             : $sd_ht{SPLITPOS2LINPOS}->{$chart_id}->{$current_chart_pos}->{$node_id}->{$i});
+	       print STDERR "  FROM.$chart_id I: $i POS: $current_chart_pos NODE: $node_id FROM: $from_linear_chart_pos\n" if $verbose;
+	       my $to_linear_chart_pos
+		  = (($i == $#roman_chars)
+		     ? $end_linpos
+		     : $sd_ht{SPLITPOS2LINPOS}->{$chart_id}->{$current_chart_pos}->{$node_id}->{($i+1)});
+	       print STDERR "  TO.$chart_id I: $i POS: $current_chart_pos NODE: $node_id FROM: $to_linear_chart_pos\n" if $verbose;
+	       my $roman_char = $roman_chars[$i];
+	       $sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}->{$to_linear_chart_pos}->{$roman_char} = 1;
+	    }
+	 } else {
+	    my $from_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos};
+	    my $to_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{($current_chart_pos+1)};
+	    # HHERE check this out
+	    my $i = 1;
+	    while (! (defined($to_linear_chart_pos))) {
+	       $i++;
+	       $to_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{($current_chart_pos+$i)};
+	    }
+	    if (defined($from_linear_chart_pos) && defined($to_linear_chart_pos)) {
+	       $sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}->{$to_linear_chart_pos}->{""} = 1
+	    } else {
+	       print STDERR "  UNDEF.$chart_id from: " 
+	                  . ((defined($from_linear_chart_pos)) ? $from_linear_chart_pos : "?")
+			  . " to: "
+	                  . ((defined($to_linear_chart_pos))   ? $to_linear_chart_pos   : "?")
+			  . "\n";
+	    }
+	 }
+      }
+      $current_chart_pos++;
+   }
+   $sd_ht{MAXLINPOS}->{$chart_id} = $sd_ht{POS2LINPOS}->{$chart_id}->{$end_chart_pos};
+}
+
+sub expand_lin_ij_roman {
+   local($this, *sd_ht, $chart_id, $lang_code, *ht) = @_;
+
+   foreach $start (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}}) {
+      foreach $end (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start}}) {
+	 foreach $roman (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start}->{$end}}) {
+	    if ($ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$roman}
+	     || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$roman}) {
+	       $this->expand_lin_ij_roman_rec(*sd_ht, $chart_id, $start, $end, $roman, $lang_code, *ht);
+	    }
+	 }
+      }
+   }
+}
+
+sub expand_lin_ij_roman_rec {
+   local($this, *sd_ht, $chart_id, $start, $end, $roman, $lang_code, *ht) = @_;
+
+   # print STDERR "  expand_lin_ij_roman_rec.$chart_id $start-$end $lang_code $roman\n";
+   return unless $ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$roman}
+              || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$roman};
+   foreach $new_end (keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$end}}) {
+      foreach $next_roman (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$end}->{$new_end}}) {
+         my $exp_roman = join("", $roman, $next_roman);
+         if ($ht{RULE_STRING}->{$lang_code}->{$exp_roman}
+          || $ht{RULE_STRING}->{""}->{$exp_roman}) {
+            $sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start}->{$new_end}->{$exp_roman} = 1;
+	    # print STDERR "  Expansion ($start-$new_end) $exp_roman\n";
+         }
+         if ($ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$exp_roman}
+          || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$exp_roman}) {
+	    $this->expand_lin_ij_roman_rec(*sd_ht, $chart_id, $start, $new_end, $exp_roman, $lang_code, *ht);
+         }
+      }
+   }
+}
+
+sub trace_string_distance {
+   local($this, *sd_ht, $chart1_id, $chart2_id, $control, $line_number, $cost) = @_;
+
+   my $chart_comb_id = join("/", $chart1_id, $chart2_id);
+   return "mismatch" if $sd_ht{MISMATCH}->{$chart_comb_id};
+   my $chart1_end = $sd_ht{MAXLINPOS}->{$chart1_id};
+   my $chart2_end = $sd_ht{MAXLINPOS}->{$chart2_id};
+   my $verbose = ($control =~ /verbose/);
+   my $chunks_p = ($control =~ /chunks/);
+   my @traces = ();
+   my @s1_s = ();
+   my @s2_s = ();
+   my @e1_s = ();
+   my @e2_s = ();
+   my @r1_s = ();
+   my @r2_s = ();
+   my @ic_s = ();
+
+   # print STDERR "trace_string_distance $chart1_id $chart2_id $line_number\n";
+   while ($chart1_end || $chart2_end) {
+      my $incr_cost = $sd_ht{INCR_COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+      my $prec_i = $sd_ht{PREC_I}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+      my $prec_j = $sd_ht{PREC_J}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+      if ($incr_cost || $verbose || $chunks_p) {
+         my $roman1 = $sd_ht{ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+         my $roman2 = $sd_ht{ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+         if ($verbose) {
+	    push(@traces, "$prec_i-$chart1_end/$prec_j-$chart2_end:$roman1/$roman2:$incr_cost");
+	 } else {
+	    if (defined($roman1)) {
+	       push(@traces, "$roman1/$roman2:$incr_cost");
+	    } else {
+	       $print_prec_i = (defined($prec_i)) ? $prec_i : "?";
+	       $print_prec_j = (defined($prec_j)) ? $prec_j : "?";
+	       print STDERR "  $prec_i-$chart1_end, $prec_j-$chart2_end\n";
+	    }
+	 }
+         if ($chunks_p) {
+            push(@s1_s, $prec_i);
+            push(@s2_s, $prec_j);
+            push(@e1_s, $chart1_end);
+            push(@e2_s, $chart2_end);
+            push(@r1_s, $roman1);
+            push(@r2_s, $roman2);
+            push(@ic_s, $incr_cost);
+         }
+      }
+      $chart1_end = $prec_i;
+      $chart2_end = $prec_j;
+   }
+   if ($chunks_p) {
+      my $r1 = "";
+      my $r2 = "";
+      my $tc = 0;
+      my $in_chunk = 0;
+      foreach $i ((0 .. $#ic_s)) {
+	 if ($ic_s[$i]) {
+	    $r1 = $r1_s[$i] . $r1;
+	    $r2 = $r2_s[$i] . $r2;
+	    $tc += $ic_s[$i];
+	    $in_chunk = 1;
+	 } elsif ($in_chunk) {
+	    $chunk = "$r1/$r2/$tc";
+	    $chunk .= "*" if $cost > 5;
+	    $sd_ht{N_COST_CHUNK}->{$chunk} = ($sd_ht{N_COST_CHUNK}->{$chunk} || 0) + 1;
+	    $sd_ht{EX_COST_CHUNK}->{$chunk}->{$line_number} = 1;
+            $r1 = "";
+            $r2 = "";
+            $tc = 0;
+	    $in_chunk = 0;
+	 }
+      }
+      if ($in_chunk) {
+	 $chunk = "$r1/$r2/$tc";
+	 $chunk .= "*" if $cost > 5;
+	 $sd_ht{N_COST_CHUNK}->{$chunk} = ($sd_ht{N_COST_CHUNK}->{$chunk} || 0) + 1;
+	 $sd_ht{EX_COST_CHUNK}->{$chunk}->{$line_number} = 1;
+      }
+   } else {
+      return join(" ", reverse @traces);
+   }
+}
+
+sub right_context_match {
+   local($this, $right_context_rule, *sd_ht, $chart_id, $start_pos) = @_;
+   
+   return 1 if $right_context_rule eq "";
+   if (($right_context_item, $right_context_rest) = ($right_context_rule =~ /^\[([^\[\]]*)\]*(.*)$/)) {
+      my $guarded_right_context_item = $right_context_item;
+      $guarded_right_context_item =~ s/\$/\\\$/g;
+      my @end_positions = keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start_pos}};
+      return 1 if ($#end_positions == -1)
+               && (($right_context_item eq "") 
+	        || ($right_context_item =~ /\$/));
+      foreach $end_pos (@end_positions) {
+	 my @romans = keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start_pos}->{$end_pos}};
+         foreach $roman (@romans) {
+	    if ($roman =~ /^[$guarded_right_context_item]/) {
+	       return $this->right_context_match($right_context_rest, *sd_ht, $chart_id, $end_pos);
+	    }
+	 }
+      }
+   }
+   return 0;
+}
+
+sub string_distance {
+   local($this, *sd_ht, $chart1_id, $chart2_id, $lang_code1, $lang_code2, *ht, $control) = @_;
+
+   my $verbose = ($control =~ /verbose/i);
+   my $chart_comb_id = join("/", $chart1_id, $chart2_id);
+
+   my $chart1_end_pos = $sd_ht{MAXLINPOS}->{$chart1_id};
+   my $chart2_end_pos = $sd_ht{MAXLINPOS}->{$chart2_id};
+   print STDERR "string_distance.$chart_comb_id $chart1_end_pos/$chart2_end_pos\n" if $verbose;
+   $sd_ht{COST_IJ}->{$chart_comb_id}->{0}->{0} = 0;
+   $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{0}->{0} = "";
+   $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{0}->{0} = "";
+   # HHERE
+   foreach $chart1_start ((0 .. $chart1_end_pos)) {
+      # print STDERR "  C1 $chart1_start- ($chart1_start .. $chart1_end_pos)\n";
+      my $prev_further_expansion_possible = 0;
+      my @chart1_ends = sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart1_id}->{$chart1_start}};
+      my $max_chart1_ends = (@chart1_ends) ? $chart1_ends[$#chart1_ends] : -1;
+      foreach $chart1_end (($chart1_start .. $chart1_end_pos)) {
+	 my $further_expansion_possible = ($chart1_start == $chart1_end)
+				       || defined($sd_ht{LINPOS2SPLITPOS}->{$chart1_id}->{$chart1_start})
+				       || ($chart1_end < $max_chart1_ends);
+	 my @romans1 = (($chart1_start == $chart1_end)
+			? ("")
+			: (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart1_id}->{$chart1_start}->{$chart1_end}}));
+	 if ($#romans1 == -1) {
+	    $further_expansion_possible = 1 if $prev_further_expansion_possible;
+	 } else {
+	    $prev_further_expansion_possible = 0;
+	 }
+         # print STDERR "  C1 $chart1_start-$chart1_end romans1: @romans1 {$further_expansion_possible} *l*\n";
+	 foreach $roman1 (@romans1) {
+            # print STDERR "  C1 $chart1_start-$chart1_end $roman1 {$further_expansion_possible} *?*\n";
+	    next unless $ht{RULE_STRING}->{$lang_code1}->{$roman1}
+		     || $ht{RULE_STRING}->{""}->{$roman1};
+            # print STDERR "  C1 $chart1_start-$chart1_end $roman1 {$further_expansion_possible} ***\n";
+	    foreach $lang_code1o (($lang_code1, "")) {
+	       foreach $lang_code2o (($lang_code2, "")) {
+		  my @chart2_starts = (sort { $a <=> $b } keys %{$sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_start}});
+	          foreach $chart2_start (@chart2_starts) {
+                     # print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start- (@chart2_starts)\n";
+		     foreach $chart2_end (($chart2_start .. $chart2_end_pos)) {
+                        print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start-$chart2_end\n";
+			my @romans2 = (($chart2_start == $chart2_end)
+				      ? ("")
+				      : (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart2_id}->{$chart2_start}->{$chart2_end}}));
+			foreach $roman2 (@romans2) {
+			   if ($roman1 eq $roman2) {
+                              print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start-$chart2_end $roman2 (IDENTITY)\n";
+			      my $cost = 0;
+			      my $preceding_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+			      my $combined_cost = $preceding_cost + $cost;
+			      my $old_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+			      if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+			         $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $combined_cost;
+			         push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+			         $sd_ht{PREC_I}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart1_start;
+			         $sd_ht{PREC_J}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart2_start;
+			         $sd_ht{ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman1;
+			         $sd_ht{ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman2;
+			         $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			            = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman1;
+			         $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			            = $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman2;
+				 $comb_left_roman1 = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+			         $sd_ht{INCR_COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $cost;
+			         $sd_ht{COST_RULE}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = "IDENTITY";
+			         print STDERR "  New cost $chart1_end/$chart2_end: $combined_cost (+$cost from $chart1_start/$chart2_start $roman1/$roman2)\n" if $verbose;
+			      }
+			   } else {
+	                      next unless $ht{RULE_STRING}->{$lang_code2o}->{$roman2};
+                              print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start-$chart2_end $roman2\n";
+			      next unless defined($ht{COST}->{$lang_code1o}->{$lang_code2o}->{$roman1}->{$roman2});
+		              my @cost_rule_ids = keys %{$ht{COST}->{$lang_code1o}->{$lang_code2o}->{$roman1}->{$roman2}};
+		              foreach $cost_rule_id (@cost_rule_ids) {
+			         ## check whether any context requirements are satisfied
+			         # left context rules are regular expressions
+                                 my $left_context_rule1 = $ht{LEFT1}->{$cost_rule_id};
+			         if ($left_context_rule1) {
+				    my $comb_left_roman1 = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+				    if (defined($comb_left_roman1)) {
+				       next unless $comb_left_roman1 =~ /$left_context_rule1/;
+			            } else {
+				       print STDERR "  No comb_left_roman1 value for $chart_comb_id $chart1_start,$chart2_start\n";
+				    }
+			         }
+                                 my $left_context_rule2 = $ht{LEFT2}->{$cost_rule_id};
+			         if ($left_context_rule2) {
+				    my $comb_left_roman2 = $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+				    if (defined($comb_left_roman2)) {
+				       next unless $comb_left_roman2 =~ /$left_context_rule2/;
+				    } else {
+				       print STDERR "  No comb_left_roman2 value for $chart_comb_id $chart1_start,$chart2_start\n";
+				    }
+			         }
+                                 my $right_context_rule1 = $ht{RIGHT1}->{$cost_rule_id};
+			         if ($right_context_rule1) {
+			            my $match_p = $this->right_context_match($right_context_rule1, *sd_ht, $chart1_id, $chart1_end);
+				    # print STDERR "  Match?($right_context_rule1, 1, $chart1_end) = $match_p\n";
+			            next unless $match_p;
+			         }
+                                 my $right_context_rule2 = $ht{RIGHT2}->{$cost_rule_id};
+			         if ($right_context_rule2) {
+			            my $match_p = $this->right_context_match($right_context_rule2, *sd_ht, $chart2_id, $chart2_end);
+				    # print STDERR "  Match?($right_context_rule2, 2, $chart2_end) = $match_p\n";
+			            next unless $match_p;
+			         }
+			         my $cost = $ht{COST}->{$lang_code1o}->{$lang_code2o}->{$roman1}->{$roman2}->{$cost_rule_id};
+			         my $preceding_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+			         my $combined_cost = $preceding_cost + $cost;
+			         my $old_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+			         if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+			            $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $combined_cost;
+				    push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+			            $sd_ht{PREC_I}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart1_start;
+			            $sd_ht{PREC_J}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart2_start;
+			            $sd_ht{ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman1;
+			            $sd_ht{ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman2;
+			            $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			               = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman1;
+			            $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			               = $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman2;
+				    $comb_left_roman1 = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+				    # print STDERR "  Comb-left-roman1($chart_comb_id,$chart1_end,$chart2_end) = $comb_left_roman1\n";
+			            $sd_ht{INCR_COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $cost;
+			            $sd_ht{COST_RULE}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $cost_rule_id;
+			            print STDERR "  New cost $chart1_end/$chart2_end: $combined_cost (+$cost from $chart1_start/$chart2_start $roman1/$roman2)\n" if $verbose;
+			         }
+			      }
+			   }
+			}
+		     }
+		  }
+	       }
+	    }
+	    $further_expansion_possible = 1
+	       if $ht{RULE_STRING_HAS_EXPANSION}->{$lang_code1}->{$roman1}
+	       || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$roman1};
+	    # print STDERR "  further_expansion_possible: $further_expansion_possible (lc: $lang_code1 r1: $roman1) ***\n";
+	 }
+         # print STDERR "  last C1 $chart1_start-$chart1_end (@romans1)\n" unless $further_expansion_possible;
+	 last unless $further_expansion_possible;
+         $prev_further_expansion_possible = 1 if $further_expansion_possible;
+      }
+   }
+   my $total_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end_pos}->{$chart2_end_pos};
+   unless (defined($total_cost)) {
+      $total_cost = 99.9999;
+      $sd_ht{MISMATCH}->{$chart_comb_id} = 1;
+   }
+   return $total_cost;
+}
+
+sub print_sd_ht {
+   local($this, *sd_ht, $chart1_id, $chart2_id, *OUT) = @_;
+
+   print OUT "string-distance chart:\n";
+   foreach $chart_id (($chart1_id, $chart2_id)) {
+      print OUT "SD chart $chart_id:\n";
+      foreach $from_linear_chart_pos (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}}) {
+	 foreach $to_linear_chart_pos (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}}) {
+	    foreach $roman_char (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}->{$to_linear_chart_pos}}) {
+	       print OUT "  Lnode($from_linear_chart_pos-$to_linear_chart_pos): $roman_char\n";
+	    }
+	 }
+      }
+   }
+}
+
+sub print_chart_ht {
+   local($this, *chart_ht, *OUT) = @_;
+
+   print OUT "uroman chart:\n";
+   foreach $start (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AT}}) {
+      foreach $end (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}}) {
+	 foreach $node_id (keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}}) {
+	    $roman_s = $chart_ht{NODE_ROMAN}->{$node_id};
+	    print OUT "  Node $node_id ($start-$end): $roman_s\n";
+	 }
+      }
+   }
+}
+
+sub normalize_string {
+   local($this, $s) = @_;
+
+#  $s =~ s/(\xE2\x80\x8C)//g; # delete zero width non-joiner
+   $s =~ s/(\xE2\x80[\x93-\x94])/-/g; # en-dash, em-dash
+   $s =~ s/([\x00-\x7F\xC0-\xFE][\x80-\xBF]*)\1+/$1$1/g; # shorten 3 or more occurrences of same character in a row to 2
+   $s =~ s/[ \t]+/ /g;
+
+   return $s;
+}
+
+my $string_distance_chart_id = 0;
+sub string_distance_by_chart {
+   local($this, $s1, $s2, $lang_code1, $lang_code2, *ht, *pinyin_ht, $control) = @_;
+
+   $control = "" unless defined($control);
+   %sd_ht = ();
+
+   $s1 = $this->normalize_string($s1);
+   my $lc_s1 = $utf8->extended_lower_case($s1);
+   $string_distance_chart_id++;
+   my $chart1_id = $string_distance_chart_id;
+   *chart_ht = $romanizer->romanize($lc_s1, $lang_code1, "", *ht, *pinyin_ht, 0, "return chart", $chart1_id);
+   $this->linearize_chart_points(*chart_ht, $chart1_id, *sd_ht);
+   $this->expand_lin_ij_roman(*sd_ht, $chart1_id, $lang_code1, *ht);
+
+   $s2 = $this->normalize_string($s2);
+   my $lc_s2 = $utf8->extended_lower_case($s2);
+   $string_distance_chart_id++;
+   my $chart2_id = $string_distance_chart_id;
+   *chart_ht = $romanizer->romanize($lc_s2, $lang_code2, "", *ht, *pinyin_ht, 0, "return chart", $chart2_id);
+   $this->linearize_chart_points(*chart_ht, $chart2_id, *sd_ht);
+   $this->expand_lin_ij_roman(*sd_ht, $chart2_id, $lang_code2, *ht);
+
+   my $cost = $this->string_distance(*sd_ht, $chart1_id, $chart2_id, $lang_code1, $lang_code2, *ht, $control);
+   return $cost;
+}
+
+my $n_quick_romanized_string_distance = 0;
+sub quick_romanized_string_distance_by_chart {
+   local($this, $s1, $s2, *ht, $control, $lang_code1, $lang_code2) = @_;
+
+   # my $verbose = ($s1 eq "apit") && ($s2 eq "apet");
+   # print STDERR "Start quick_romanized_string_distance_by_chart\n";
+   $s1 = lc $s1;
+   $s2 = lc $s2;
+   $control = "" unless defined($control);
+   $lang_code1 = "" unless defined($lang_code1);
+   $lang_code2 = "" unless defined($lang_code2);
+   my $cache_p = ($control =~ /cache/);
+   my $total_cost;
+   if ($cache_p) {
+      $total_cost = $ht{CACHED_QRSD}->{$s1}->{$s2};
+      if (defined($total_cost)) {
+         return $total_cost;
+      }
+   }
+   my @lang_codes1 = ($lang_code1 eq "") ? ("") : ($lang_code1, "");
+   my @lang_codes2 = ($lang_code2 eq "") ? ("") : ($lang_code2, "");
+   my $chart1_end_pos = length($s1);
+   my $chart2_end_pos = length($s2);
+   my %sd_ht = ();
+   $sd_ht{COST_IJ}->{0}->{0} = 0;
+   foreach $chart1_start ((0 .. $chart1_end_pos)) {
+      foreach $chart1_end (($chart1_start .. $chart1_end_pos)) {
+	 my $substr1 = substr($s1, $chart1_start, ($chart1_end-$chart1_start));
+	 foreach $lang_code1o (@lang_codes1) {
+	    foreach $lang_code2o (@lang_codes2) {
+	       # next unless defined($ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1});
+            }
+	 }
+	 my @chart2_starts = (sort { $a <=> $b } keys %{$sd_ht{COST_IJ}->{$chart1_start}});
+	 foreach $chart2_start (@chart2_starts) {
+	    foreach $chart2_end (($chart2_start .. $chart2_end_pos)) {
+	       my $substr2 = substr($s2, $chart2_start, ($chart2_end-$chart2_start));
+	       foreach $lang_code1o (@lang_codes1) {
+	          foreach $lang_code2o (@lang_codes2) {
+		     if ($substr1 eq $substr2) {
+			my $cost = 0;
+	                my $preceding_cost = $sd_ht{COST_IJ}->{$chart1_start}->{$chart2_start};
+			if (defined($preceding_cost)) {
+		           my $combined_cost = $preceding_cost + $cost;
+	                   my $old_cost = $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end};
+	                   if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+		              $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end} = $combined_cost;
+		              push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+		           }
+			}
+		     } else {
+	                next unless defined($ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1}->{$substr2});
+	                my @cost_rule_ids = keys %{$ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1}->{$substr2}};
+	                my $best_cost = 99.99;
+	                foreach $cost_rule_id (@cost_rule_ids) {
+		           my $cost = $ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1}->{$substr2}->{$cost_rule_id};
+                           my $left_context_rule1 = $ht{LEFT1}->{$cost_rule_id};
+			   next if $left_context_rule1
+			        && (! (substr($s1, 0, $chart1_start) =~ /$left_context_rule1/));
+                           my $left_context_rule2 = $ht{LEFT2}->{$cost_rule_id};
+			   next if $left_context_rule2
+			        && (! (substr($s2, 0, $chart2_start) =~ /$left_context_rule2/));
+                           my $right_context_rule1 = $ht{RIGHT1}->{$cost_rule_id};
+			   my $right_context1 = substr($s1, $chart1_end);
+			   next if $right_context_rule1
+			        && (! (($right_context1 =~ /^$right_context_rule1/)
+			            || (($right_context_rule1 =~ /^\[[^\[\]]*\$/)
+			             && ($right_context1 eq ""))));
+                           my $right_context_rule2 = $ht{RIGHT2}->{$cost_rule_id};
+			   my $right_context2 = substr($s2, $chart2_end);
+			   next if $right_context_rule2
+			        && (! (($right_context2 =~ /^$right_context_rule2/)
+			            || (($right_context_rule2 =~ /^\[[^\[\]]*\$/)
+			             && ($right_context2 eq ""))));
+		           $best_cost = $cost if $cost < $best_cost;
+	                   my $preceding_cost = $sd_ht{COST_IJ}->{$chart1_start}->{$chart2_start};
+		           my $combined_cost = $preceding_cost + $cost;
+	                   my $old_cost = $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end};
+	                   if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+		              $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end} = $combined_cost;
+		              push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+		           }
+		        }
+		     }
+		  }
+	       }
+	    }
+	 }
+      }
+   }
+   $total_cost = $sd_ht{COST_IJ}->{$chart1_end_pos}->{$chart2_end_pos};
+   $total_cost = 99.99 unless defined($total_cost);
+   $ht{CACHED_QRSD}->{$s1}->{$s2} = $total_cost if $cache_p;
+   $n_quick_romanized_string_distance++;
+   return $total_cost;
+}
+
+sub get_n_quick_romanized_string_distance {
+   return $n_quick_romanized_string_distance;
+}
+
+1;
+

uroman/tarballs/uroman-v1.0.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:912655beef069e5abb43c8fc4c3c4428fd0af6f4a1697accc98277933d3e1ee5
+size 440252

uroman/tarballs/uroman-v1.1.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df990f6096a10e093ac5f28c2b86d5ef9e9098ef7472855843f9a841bb3b963d
+size 507234

uroman/tarballs/uroman-v1.2.4.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77d707f3c17d5c45869b80fe71caee6023d1d9949ccffb446626f374605a25e2
+size 503690

uroman/tarballs/uroman-v1.2.5.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2e9044afff8b4483f43a99b1fb1279889336760d76245ee93f300e660a46660
+size 575581

uroman/tarballs/uroman-v1.2.6.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02f6f73b067b972a8f7d408da2f9b22741629af67f55b2ea768d11710fbf40a4
+size 567522

uroman/tarballs/uroman-v1.2.7.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbb51506ed3ea6dcb902c824e62bea39b3741f6526564ba05d6e0083d8d876e5
+size 566800

uroman/tarballs/uroman-v1.2.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c69e56d9c5eea9416ae00ca4dd859a1ef5129c1867778b66ad2f811f0fd33c9
+size 494625

uroman/test/multi-script.txt

@@ -0,0 +1,32 @@
+::lcode deu Grüße aus Bordeaux
+::lcode tur İstanbul, Türkiye'de yer alan şehir ve ülkenin 81 ilinden biri.
+::lcode eng ⠠⠺⠑⠀⠓⠕⠇⠙⠀⠘⠮⠀⠞⠗⠥⠹⠎⠀⠞⠕⠀⠆⠀⠎⠑⠇⠋⠤⠑⠧⠊⠙⠢⠞⠂⠀⠞⠀⠁⠇⠇⠀⠍⠑⠝⠀⠜⠑⠀⠉⠗⠂⠞⠫⠀⠑⠟⠥⠁⠇⠂⠀⠞⠀⠮⠽⠀⠜⠑⠀⠑⠝⠙⠪⠫⠀⠃⠽⠀⠸⠮⠀⠠⠉⠗⠑⠁⠞⠕⠗⠀⠾⠀⠉⠻⠞⠁⠔⠀⠥⠝⠁⠇⠊⠑⠝⠁⠃⠇⠑⠀⠠⠐⠗⠎⠂⠀⠞⠀⠁⠍⠰⠛⠀⠘⠮⠀⠜⠑⠀⠠⠇⠊⠋⠑⠂⠀⠠⠇⠊⠃⠻⠞⠽⠀⠯⠀⠮⠀⠏⠥⠗⠎⠥⠊⠞⠀⠷⠀⠠⠓⠁⠏⠏⠊⠰⠎⠲
+::lcode ell Το Λος Άντζελες (στα ισπανικά Los Angeles = Οι Άγγελοι) ή στην Αμερικανική αργκό L.A., ελ έι) είναι η δεύτερη μεγαλύτερη πόλη των Ηνωμένων Πολιτειών από άποψη πληθυσμού, καθώς και ένα από τα σημαντικότερα οικονομικά, πολιτιστικά επιστημονικά και ψυχαγωγικά κέντρα του κόσμου.
+::lcode rus Герма́ния (нем. Deutschland), официальное название — Федерати́вная Респу́блика Герма́ния (нем. Bundesrepublik Deutschland), ФРГ (нем. BRD) — государство в Западной Европе. Площадь территории — 357 021 км². Численность населения по переписи 2011 года — более 80 миллионов человек. [2][6].
+::lcode ukr Володи́мир Олекса́ндрович Зеле́нський (нар. 25 січня 1978, Кривий Ріг) — український державний діяч, політик, шоумен, актор, комік, режисер, продюсер та сценарист, шостий Президент України з 20 травня 2019 року.
+::lcode srp Сва људска бића рађају се слободна и једнака у достојанству и правима. Она су обдарена разумом и свешћу и треба једни према другима да поступају у духу братства.
+::lcode ara كندا (بالإنجليزية: Canada) هي دولة في أمريكا الشمالية تتألف من 10 مقاطعات وثلاثة أقاليم. تقع في القسم الشمالي من القارة وتمتد من المحيط الأطلسي في الشرق إلى المحيط الهادئ في الغرب وتمتد شمالاً في المحيط المتجمد الشمالي. كندا هي البلد الثاني عالمياً من حيث المساحة الكلية. كما أن حدود كندا المشتركة مع الولايات المتحدة من الجنوب والشمال الغربي هي الأطول في العالم.
+::lcode fas کالیفرنیا (به انگلیسی: California) ایالتی در غرب آمریکا بر کرانهٔ اقیانوس آرام است. مرکز آن ساکرامنتو و شهرهای مهم آن لس‌آنجلس، سن دیگو، سن خوزه و سان‌فرانسیسکو هستند.همچنین این ایالت پر جمعیت ترین ایالت امریکا است.
+::lcode uig ئامېرىكا قوشما شتاتلىرى بولسا شىمالىي ئامېرىكاغا جايلاشقان بىر دۆلەت. ئۇنىڭ پايتەختى بولسا ۋاشىنگتون، ئەڭ چوڭ شەھىرى بولسا نيۇيورك شەھىرى. دۆلەت تىلى بولسا ئېنگلىزتىلى. ھازىرقى زۇڭتۇڭ باراك ئوباما. بۇ دۆلەت ئەسلىدە ئەنگىلىيەنىڭ مۇستەملىكىسى بولۇپ ۋاشىنگىتوننىڭ رەھپەرلىكىدە 1776 يىلى 7 ئاينىڭ 4 كۇنى مۇستەقىل بولغان، يەر مەيدانى 9 مىلىيون 826 مىڭ 630 كۋادىرات كلومېتىر، نوپۇسى 306 مىللىيون 142 مىڭ، بۇلارنىڭ ئاسساسلىق دىنى خرىستىئان دىنى.
+::lcode amh ኢትዮጵያ ከዓለም ሶስቱ ትልቅ የአብርሃም ሀይማኖቶች ጋር ታሪካዊ ግንኙነት አላት።
+::lcode hin कैलिफ़ोर्निया शब्द का पहला अर्थ था जो क्षेत्र जहाँ आज बाहा कैलिफ़ोर्निया प्रायद्वीप, नेवाडा, यूटा और एरिज़ोना, नया मेक्सिको, और वायोमिंग के कई विभाग स्थित हैं।
+::lcode mar लंडन (इंग्लिश: London ) हे इंग्लंडचे व युनायटेड किंग्डमचे राजधानीचे व सर्वात मोठे शहर तसेच युरोपियन संघामधील सर्वात मोठे महान���र क्षेत्र आहे.
+::lcode nep यसको उचाइ समुन्द्र सतहबाट ८,८४८ मीटर (२९,०२८ फीट) छ। यो नेपालको सोलुखुम्बु जिल्लाको खुम्जुङ्ग गा. वि. स. मा पर्छ ।
+::lcode tam தமிழ்நாடு (Tamil Nadu) இந்தியாவின் 29 மாநிலங்களில் ஒன்றாகும். தமிழ்நாடு, தமிழகம் என்றும் பரவலாக அழைக்கப்படுகிறது.
+::lcode mal ഇന്ത്യയുടെ തെക്കുപടിഞ്ഞാറെ അറ്റത്തുള്ള സംസ്ഥാനമാണ് കേരളം.
+::lcode ori ଓଡ଼ିଶା ଭାରତର ପୂର୍ବ ଉପକୂଳରେ ଥିବା ଏକ ପ୍ରଶାସନିକ ରାଜ୍ୟ । ଏହାର ଉତ୍ତର-ପୂର୍ବରେ ପଶ୍ଚିମବଙ୍ଗ, ଉତ୍ତରରେ ଝାଡ଼ଖଣ୍ଡ, ପଶ୍ଚିମ ଓ ଉତ୍ତର-ପଶ୍ଚିମରେ ଛତିଶଗଡ଼, ଦକ୍ଷିଣ ଓ ଦକ୍ଷିଣ-ପଶ୍ଚିମରେ ଆନ୍ଧ୍ରପ୍ରଦେଶ ଅବସ୍ଥିତ । ଏହା ଆୟତନ ହିସାବରେ ନବମ ଓ ଜନସଂଖ୍ୟା ହିସାବରେ ଏଗାରତମ ରାଜ୍ୟ । ଓଡ଼ିଆ ଭାଷା ରାଜ୍ୟର ସରକାରୀ ଭାଷା । ୨୦୦୧ ଜନଗଣନା ଅନୁସାରେ ରାଜ୍ୟର ପ୍ରାୟ ୩୩.୨ ନିୟୁତ ଲୋକ ଓଡ଼ିଆ ଭାଷା ବ୍ୟବହାର କରନ୍ତି । 
+::lcode zho 加拿大在一万四千年前即有原住民在此生活。
+::lcode heb כֹּל עוֹד בַּלֵּבָב פְּנִימָה נֶפֶשׁ יְהוּדִי הוֹמִיָּה וּלְפַאֲתֵי מִזְרָח, קָדִימָה, עַיִן לְצִיּוֹן צוֹפִיָּה, עוֹד לֹא אָבְדָה תִּקְוָתֵנוּ, הַתִּקְוָה בַּת שְׁנוֹת אַלְפַּיִם לִהְיוֹת עַם חָפְשִׁי בְּאַרְצֵנוּ, אֶרֶץ צִיּוֹן וִירוּשָׁלַיִם.
+::lcode yid דווקא איז אן העברעישער זשורנאל וואס באשרייבט די יידיש־שפראכיקע קולטור. עס איז דערשינען געווארן תמוז ה'תשס"ז (יולי 2006).
+::lcode hye Տալնոեի շրջան (ուկր.՝ Тальнівський район), շրջան Ուկրաինայի Չերկասիի մարզում։ Ստեղծվել է 1923 թվականին։ Վարչական կենտրոնը՝ Տալնոե։ Աշխարհագրությունը Շրջանի տարածքի մակերեսը կազմում է 917 կմ²։ Բնակչություն
+::lcode tai มีประเทศอิสระ 2 ประเทศ คือ ซานมารีโนและนครรัฐวาติกัน เป็นดินแดนที่ล้อมรอบไปด้วยพื้นที่ของอิตาลี ในขณะที่เมืองกัมปีโอเนดีตาเลีย เป็นดินแดนส่วนแยกของอิตาลีที่ถูกล้อมรอบด้วยพื้นที่ประเทศสวิตเซอร์แลนด์
+북쪽에는 인도네시아와 동티모르, 파푸아 뉴기니, 북동쪽에는 솔로몬 제도와 바누아투, 누벨칼레도니, 그리고 남동쪽에는 뉴질랜드가 있다.
+ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸು ಇಂದೆನ್ನ ಹೃದಯದಲಿ ನಿತ್ಯವೂ ಅವತರಿಪ ಸತ್ಯಾವತಾರ ಮಣ್ಣಾಗಿ ಮರವಾಗಿ ಮಿಗವಾಗಿ ಕಗವಾಗೀ... ಮಣ್ಣಾಗಿ ಮರವಾಗಿ ಮಿಗವಾಗಿ ಕಗವಾಗಿ ಭವ ಭವದಿ ಭತಿಸಿಹೇ ಭವತಿ ದೂರ ನಿತ್ಯವೂ ಅವತರಿಪ ಸತ್ಯಾವತಾರ || ಬಾ ಇಲ್ಲಿ ||
+ვეპხის ტყაოსანი შოთა რუსთაველი ღმერთსი შემვედრე, ნუთუ კვლა დამხსნას სოფლისა შრომასა, ცეცხლს, წყალსა და მიწასა, ჰაერთა თანა მრომასა; მომცნეს ფრთენი და აღვფრინდე, მივჰხვდე მას ჩემსა ნდომასა, დღისით და ღამით ვჰხედვიდე მზისა ელვათა კრთომაასა.
+᚛ᚐᚅᚋ ᚋᚖᚂᚓᚌᚖᚋᚏᚔᚇ ᚋᚐᚉᚔ ᚍᚓᚉᚒᚋᚓᚅ᚜
+ᛁᚳ᛫ᛗᚨᚷ᛫ᚷᛚᚨᛋ᛫ᛖᚩᛏᚪᚾ᛫ᚩᚾᛞ᛫ᚻᛁᛏ᛫ᚾᛖ᛫ᚻᛖᚪᚱᛗᛁᚪᚧ᛫ᛗᛖ᛬
+𓊪𓏏𓍯𓃭𓐝𓇌𓋴
+チェコスロバキア
+ལྷ་ས་གྲ���ང་ཁྱེར
+ᓵᓕ ᓴᕕᐊᕐᔪᒃ ᐃᒻᒥᓂᒃ ᓂᓪᓕᕈᑎᖃᓲᖑᕗᖅ ᑕᐃᑦᓱᒪᓂᑕᑦᓴᔭᐅᓂᕋᕐᓱᓂ. ᐃᒻᒥᓂᓪᓗᑕᐅᖅ ᓂᓪᓕᕈᑎᖃᓱᖑᒻᒥᓱᓂ ᐅᓪᓗᒥᓂᑕᑦᓴᔭᐅᓂᕋᕐᓱᓂ.
+ⴰⵎⴰⴳⵔⴰⴷ 1 ⴰⵔ ⴷ ⵜⵜⵍⴰⵍⴰⵏ ⵎⵉⴷⴷⵏ ⴳⴰⵏ ⵉⵍⴻⵍⵍⵉⵜⵏ ⵎⴳⴰⴷⴷⴰⵏ ⵖ ⵡⴰⴷⴷⵓⵔ ⴷ ⵉⵣⵔⴼⴰⵏ, ⵢⵉⵍⵉ ⴰⴽⵯ ⴷⴰⵔⵙⵏ ⵓⵏⵍⵍⵉ ⴷ ⵓⴼⵔⴰⴽ, ⵉⵍⵍⴰ ⴼⵍⵍⴰ ⵙⵏ ⴰⴷ ⵜⵜⵎⵢⴰⵡⴰⵙⵏ ⵏⴳⵔⴰⵜⵙⵏ ⵙ ⵜⴰⴳⵎⴰⵜ.

uroman/test/multi-script.uroman-ref.txt

@@ -0,0 +1,32 @@
+::lcode deu Gruesse aus Bordeaux
+::lcode tur Istanbul, Tuerkiye'de yer alan shehir ve uelkenin 81 ilinden biri.
+::lcode eng We hold ⠘e truos to ; self-evid⠢t, t all men aee cr,te equal, t ey aee endoee by ⠸e Creator u cita⠔ unalienable ⠠⠐rs, t amg ⠘e aee Life, Libity ⠯ e pursuit a Happis.
+::lcode ell To Los Andzeles (sta ispanika Los Angeles = Oi Angeloi) e sten Amerikanike arngo L.A., el ei) einai e deutere megalutere pole ton Enomenon Politeion apo apopse plethysmou, kathos kai ena apo ta semandikotera oikonomika, politistika epistemonika kai psychagogika kendra tou kosmou.
+::lcode rus Germaniya (nem. Deutschland), ofitsialnoe nazvanie — Federativnaya Respublika Germaniya (nem. Bundesrepublik Deutschland), FRG (nem. BRD) — gosudarstvo v Zapadnoi Evrope. Ploshchad territorii — 357 021 km². Chislennost naseleniya po perepisi 2011 goda — bolee 80 millionov chelovek. [2][6].
+::lcode ukr Volodimir Oleksandrovich Zelensky (nar. 25 sichnya 1978, Krivy Rig) — ukrayinsky derzhavny diyach, politik, shoumen, aktor, komik, rezhiser, prodyuser ta stsenarist, shosty Prezident Ukrayini z 20 travnya 2019 roku.
+::lcode srp Sva ljudska bitsha radjaju se slobodna i jednaka u dostojanstvu i pravima. Ona su obdarena razumom i sveshtshu i treba jedni prema drugima da postupaju u dukhu bratstva.
+::lcode ara knda (balinjlyzya: Canada) hy dwla fy amryka alshmalya ttalf mn 10 mqat'at wthlatha aqalym. tq' fy alqsm alshmaly mn alqara wtmtd mn almhyt alatlsy fy alshrq ila almhyt alhadye fy alghrb wtmtd shmalan fy almhyt almtjmd alshmaly. knda hy albld althany 'almyan mn hyth almsaha alklya. kma an hdwd knda almshtrka m' alwlayat almthda mn aljnwb walshmal alghrby hy alatwl fy al'alm.
+::lcode fas kalifrnia (bh anglisi: California) ialti dr ghrb amrika br kranh' aqianws aram ast. mrkz an sakramntw w shhrhai mhm an lsanjls, sn digw, sn khwzh w sanfransiskw hstnd.hmtchnin in ialt pr jm'it trin ialt amrika ast.
+::lcode uig yeameraka qwshma shtatlara bwlsa shamalay yeamerakagha jaylashqan bar doelaet. yeunang paytaekhta bwlsa vashangtwn, yeaeng tchwng shaehara bwlsa nyuywrk shaehara. doelaet tala bwlsa yeenglaztala. hazarqa zungtung barak yewbama. bu doelaet yeaesladae yeaengalayaenang mustaemlakasa bwlup vashangatwnnang raehpaerlakadae 1776 yala 7 yeaynang 4 kuna mustaeqal bwlghan, yaer maeydana 9 malaywn 826 mang 630 kvadarat klwmetar, nwpusa 306 mallaywn 142 mang, bularnang yeassaslaq dana khrastayean dana.
+::lcode amh iteyopheyaa kaaalame sosetu teleqe yaaberehaame hayemaanotoche gaare taarikaawi genenyunate alaate.
+::lcode hin kailiphorniyaa shabda kaa pahalaa artha thaa jo kssetra jahaam aaj baahaa kailiphorniyaa praayadviip, nevaaddaa, yuuttaa aur erijonaa, nayaa meksiko, aur vaayomimga ke kaii vibhaag sthit haim.
+::lcode mar lamddan (imglish: London ) he imglamddace va yunaayattedd kimgddamace raajadhaaniice va sarvaat motthe shahar tasec yuropiyan samghaamadhiil sarvaat motthe mahaanagar kssetra aahe.
+::lcode nep yasako ucaai samundra satahabaatt 8,848 miittar (29,028 phiitt) cha. yo nepaalako solukhumbu jillaako khumjungga gaa. vi. sa. maa parcha .
+::lcode tam tamilnaadu (Tamil Nadu) intiyaavin 29 maanilangkalil onraakum. tamilnaadu, tamilakam enrum paravalaaka alaikkappadukiratu.
+::lcode mal intyayutte tekkupattinynyaarre arrrrattulllla samsthaanamaann keerallam.
+::lcode ori oddishaa bhaaratara puurba upakuullare thibaa eka prashaasanika raajya . ehaara uttara-puurbare pashcimabangga, uttarare jhaaddakhanndda, pashcima o uttara-pashcimare chatishagadda, dakssinna o dakssinna-pashcimare aandhrapradesha abasthita . ehaa aayatana hisaabare nabama o janasamkhyaa hisaabare egaaratama raajya . oddiaa bhaassaa raajyara sarakaarii bhaassaa . 2001 janagannanaa anusaare raajyara praaya 33.2 niyuta loka oddiaa bhaassaa byabahaara karanti . 
+::lcode zho jianadazai14000nianqianjiyouyuanzhuminzaicishenghuo. 
+::lcode heb kol 'od balevav penimah nefesh yehudi homiyah ulefa'ate mizerach, qadimah, 'ayin letsiyon tsofiyah, 'od lo avedah tiqvatenu, hatiqvah bat shenot 'alepayim liheyot 'am chafeshiy be'aretsenu, erets tsiyon virushalayim.
+::lcode yid dvvqa ayz an h'vr'ysh'r zshvrnal vvas vashryyvt dy yydysh-shfrakyq' qvltvr. 's ayz d'rshyn'n g'vvarn tmvz h'tshs"z (yvly 2006).
+::lcode hye Talnoei shrjan (ukr., Talnivsky raion), shrjan Ukrainayi Cherkasii marzum. Steghtsvel e 1923 tvakanin. Varchakan kentrone, Talnoe. Ashkharhagrutyune Shrjani taratski makerese kazmum e 917 km². Bnakchutyun
+::lcode tai miipratesisra 2 prates kuee saanmaariinolaeankrratwaatikan peondindaentiilomrobpaidwypueentiikongitaalii naiknatiimeueengkampiionediitaaleiiy peondindaenswnyaekkongitaaliitiituuklomrobdwypueentiipratesswitserlaend
+bugjjogeneun indonesiawa dongtimoreu, papua nyugini, bugdongjjogeneun solromon jedowa banuatu, nubelkalredoni, geurigo namdongjjogeneun nyujilraendeuga issda.
+baa illi sambhavisu imdenna hrdayadali nityavuu avataripa satyaavataara mannnnaagi maravaagi migavaagi kagavaagii... mannnnaagi maravaagi migavaagi kagavaagi bhava bhavadi bhatisihee bhavati duura nityavuu avataripa satyaavataara || baa illi ||
+vepxis tqaosani shota rustaveli ghmertsi shemvedre, nutu kvla damxsnas sophlisa shromasa, tsetsxls, tsqalsa da mitsasa, haerta tana mromasa; momtsnes phrteni da aghvphrinde, mivhxvde mas chemsa ndomasa, dghisit da ghamit vhxedvide mzisa elvata krtomaasa.
+anm moilegoimrid maki vekumen
+ic mag glas eotan ond hit ne hearmiath me.
+ptolmys
+chekosurobakia
+lha·sa·grong·khyer
+saali safiaryok imminik nillirotiqasoongofoq taitsomanitatsayaonirarsoni. imminillotaoq nillirotiqasongommisoni ollominitatsayaonirarsoni.
+amagrad 1 ar d ttlalan middn gan ilellitn mgaddan gh waddur d izrfan, yili ak darsn unlli d ufrak, illa flla sn ad ttmyawasn ngratsn s tagmat.

uroman/test/string-similarity-test-input.txt

@@ -0,0 +1,7 @@
+trap	strap
+colour	color
+labeling	labelling
+organisation	organization
+Philadelphia	Filadelfia
+Vladimir	Volodymyr
+Moskva	Moskvoy