MultiMed

Running

App Files Files Community

Raghavan1988 commited on Nov 6, 2023

Commit

f86940b

•

1 Parent(s): 579282f

adding language list from seamless_m4t

Browse files

Files changed (2) hide show

app.py +9 -0
lang_list.py +402 -0

app.py CHANGED Viewed

@@ -9,6 +9,15 @@ import json
 from decouple import Config
 from transformers import AutoProcessor, SeamlessM4TModel
 config = Config('.env')
 DEFAULT_TARGET_LANGUAGE = "English"

 from decouple import Config
 from transformers import AutoProcessor, SeamlessM4TModel
+from lang_list import (
+    LANGUAGE_NAME_TO_CODE,
+    S2ST_TARGET_LANGUAGE_NAMES,
+    S2TT_TARGET_LANGUAGE_NAMES,
+    T2TT_TARGET_LANGUAGE_NAMES,
+    TEXT_SOURCE_LANGUAGE_NAMES,
+    LANG_TO_SPKR_ID,
+)
 config = Config('.env')
 DEFAULT_TARGET_LANGUAGE = "English"

lang_list.py ADDED Viewed

	@@ -0,0 +1,402 @@

+# Language dict
+language_code_to_name = {
+    "afr": "Afrikaans",
+    "amh": "Amharic",
+    "arb": "Modern Standard Arabic",
+    "ary": "Moroccan Arabic",
+    "arz": "Egyptian Arabic",
+    "asm": "Assamese",
+    "ast": "Asturian",
+    "azj": "North Azerbaijani",
+    "bel": "Belarusian",
+    "ben": "Bengali",
+    "bos": "Bosnian",
+    "bul": "Bulgarian",
+    "cat": "Catalan",
+    "ceb": "Cebuano",
+    "ces": "Czech",
+    "ckb": "Central Kurdish",
+    "cmn": "Mandarin Chinese",
+    "cym": "Welsh",
+    "dan": "Danish",
+    "deu": "German",
+    "ell": "Greek",
+    "eng": "English",
+    "est": "Estonian",
+    "eus": "Basque",
+    "fin": "Finnish",
+    "fra": "French",
+    "gaz": "West Central Oromo",
+    "gle": "Irish",
+    "glg": "Galician",
+    "guj": "Gujarati",
+    "heb": "Hebrew",
+    "hin": "Hindi",
+    "hrv": "Croatian",
+    "hun": "Hungarian",
+    "hye": "Armenian",
+    "ibo": "Igbo",
+    "ind": "Indonesian",
+    "isl": "Icelandic",
+    "ita": "Italian",
+    "jav": "Javanese",
+    "jpn": "Japanese",
+    "kam": "Kamba",
+    "kan": "Kannada",
+    "kat": "Georgian",
+    "kaz": "Kazakh",
+    "kea": "Kabuverdianu",
+    "khk": "Halh Mongolian",
+    "khm": "Khmer",
+    "kir": "Kyrgyz",
+    "kor": "Korean",
+    "lao": "Lao",
+    "lit": "Lithuanian",
+    "ltz": "Luxembourgish",
+    "lug": "Ganda",
+    "luo": "Luo",
+    "lvs": "Standard Latvian",
+    "mai": "Maithili",
+    "mal": "Malayalam",
+    "mar": "Marathi",
+    "mkd": "Macedonian",
+    "mlt": "Maltese",
+    "mni": "Meitei",
+    "mya": "Burmese",
+    "nld": "Dutch",
+    "nno": "Norwegian Nynorsk",
+    "nob": "Norwegian Bokm\u00e5l",
+    "npi": "Nepali",
+    "nya": "Nyanja",
+    "oci": "Occitan",
+    "ory": "Odia",
+    "pan": "Punjabi",
+    "pbt": "Southern Pashto",
+    "pes": "Western Persian",
+    "pol": "Polish",
+    "por": "Portuguese",
+    "ron": "Romanian",
+    "rus": "Russian",
+    "slk": "Slovak",
+    "slv": "Slovenian",
+    "sna": "Shona",
+    "snd": "Sindhi",
+    "som": "Somali",
+    "spa": "Spanish",
+    "srp": "Serbian",
+    "swe": "Swedish",
+    "swh": "Swahili",
+    "tam": "Tamil",
+    "tel": "Telugu",
+    "tgk": "Tajik",
+    "tgl": "Tagalog",
+    "tha": "Thai",
+    "tur": "Turkish",
+    "ukr": "Ukrainian",
+    "urd": "Urdu",
+    "uzn": "Northern Uzbek",
+    "vie": "Vietnamese",
+    "xho": "Xhosa",
+    "yor": "Yoruba",
+    "yue": "Cantonese",
+    "zlm": "Colloquial Malay",
+    "zsm": "Standard Malay",
+    "zul": "Zulu",
+}
+LANGUAGE_NAME_TO_CODE = {v: k for k, v in language_code_to_name.items()}
+# Source langs: S2ST / S2TT / ASR don't need source lang
+# T2TT / T2ST use this
+text_source_language_codes = [
+    "afr",
+    "amh",
+    "arb",
+    "ary",
+    "arz",
+    "asm",
+    "azj",
+    "bel",
+    "ben",
+    "bos",
+    "bul",
+    "cat",
+    "ceb",
+    "ces",
+    "ckb",
+    "cmn",
+    "cym",
+    "dan",
+    "deu",
+    "ell",
+    "eng",
+    "est",
+    "eus",
+    "fin",
+    "fra",
+    "gaz",
+    "gle",
+    "glg",
+    "guj",
+    "heb",
+    "hin",
+    "hrv",
+    "hun",
+    "hye",
+    "ibo",
+    "ind",
+    "isl",
+    "ita",
+    "jav",
+    "jpn",
+    "kan",
+    "kat",
+    "kaz",
+    "khk",
+    "khm",
+    "kir",
+    "kor",
+    "lao",
+    "lit",
+    "lug",
+    "luo",
+    "lvs",
+    "mai",
+    "mal",
+    "mar",
+    "mkd",
+    "mlt",
+    "mni",
+    "mya",
+    "nld",
+    "nno",
+    "nob",
+    "npi",
+    "nya",
+    "ory",
+    "pan",
+    "pbt",
+    "pes",
+    "pol",
+    "por",
+    "ron",
+    "rus",
+    "slk",
+    "slv",
+    "sna",
+    "snd",
+    "som",
+    "spa",
+    "srp",
+    "swe",
+    "swh",
+    "tam",
+    "tel",
+    "tgk",
+    "tgl",
+    "tha",
+    "tur",
+    "ukr",
+    "urd",
+    "uzn",
+    "vie",
+    "yor",
+    "yue",
+    "zsm",
+    "zul",
+]
+TEXT_SOURCE_LANGUAGE_NAMES = sorted([language_code_to_name[code] for code in text_source_language_codes])
+# Target langs:
+# S2ST / T2ST
+s2st_target_language_codes = [
+    "eng",
+    "arb",
+    "ben",
+    "cat",
+    "ces",
+    "cmn",
+    "cym",
+    "dan",
+    "deu",
+    "est",
+    "fin",
+    "fra",
+    "hin",
+    "ind",
+    "ita",
+    "jpn",
+    "kor",
+    "mlt",
+    "nld",
+    "pes",
+    "pol",
+    "por",
+    "ron",
+    "rus",
+    "slk",
+    "spa",
+    "swe",
+    "swh",
+    "tel",
+    "tgl",
+    "tha",
+    "tur",
+    "ukr",
+    "urd",
+    "uzn",
+    "vie",
+]
+S2ST_TARGET_LANGUAGE_NAMES = sorted([language_code_to_name[code] for code in s2st_target_language_codes])
+# S2TT / ASR
+S2TT_TARGET_LANGUAGE_NAMES = TEXT_SOURCE_LANGUAGE_NAMES
+# T2TT
+T2TT_TARGET_LANGUAGE_NAMES = TEXT_SOURCE_LANGUAGE_NAMES
+LANG_TO_SPKR_ID = {
+    "arb": [
+        0
+    ],
+    "ben": [
+        2,
+        1
+    ],
+    "cat": [
+        3
+    ],
+    "ces": [
+        4
+    ],
+    "cmn": [
+        5
+    ],
+    "cym": [
+        6
+    ],
+    "dan": [
+        7,
+        8
+    ],
+    "deu": [
+        9
+    ],
+    "eng": [
+        10
+    ],
+    "est": [
+        11,
+        12,
+        13
+    ],
+    "fin": [
+        14
+    ],
+    "fra": [
+        15
+    ],
+    "hin": [
+        16
+    ],
+    "ind": [
+        17,
+        24,
+        18,
+        20,
+        19,
+        21,
+        23,
+        27,
+        26,
+        22,
+        25
+    ],
+    "ita": [
+        29,
+        28
+    ],
+    "jpn": [
+        30
+    ],
+    "kor": [
+        31
+    ],
+    "mlt": [
+        32,
+        33,
+        34
+    ],
+    "nld": [
+        35
+    ],
+    "pes": [
+        36
+    ],
+    "pol": [
+        37
+    ],
+    "por": [
+        38
+    ],
+    "ron": [
+        39
+    ],
+    "rus": [
+        40
+    ],
+    "slk": [
+        41
+    ],
+    "spa": [
+        42
+    ],
+    "swe": [
+        43,
+        45,
+        44
+    ],
+    "swh": [
+        46,
+        48,
+        47
+    ],
+    "tel": [
+        49
+    ],
+    "tgl": [
+        50
+    ],
+    "tha": [
+        51,
+        54,
+        55,
+        52,
+        53
+    ],
+    "tur": [
+        58,
+        57,
+        56
+    ],
+    "ukr": [
+        59
+    ],
+    "urd": [
+        60,
+        61,
+        62
+    ],
+    "uzn": [
+        63,
+        64,
+        65
+    ],
+    "vie": [
+        66,
+        67,
+        70,
+        71,
+        68,
+        69
+    ]
+}