Upload utils.py with huggingface_hub
Browse files
utils.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
LANGUAGE_CODE_TO_NAME = {
|
| 2 |
+
"afr": "Afrikaans",
|
| 3 |
+
"amh": "Amharic",
|
| 4 |
+
"arb": "Modern Standard Arabic",
|
| 5 |
+
"ary": "Moroccan Arabic",
|
| 6 |
+
"arz": "Egyptian Arabic",
|
| 7 |
+
"asm": "Assamese",
|
| 8 |
+
"ast": "Asturian",
|
| 9 |
+
"azj": "North Azerbaijani",
|
| 10 |
+
"bel": "Belarusian",
|
| 11 |
+
"ben": "Bengali",
|
| 12 |
+
"bos": "Bosnian",
|
| 13 |
+
"bul": "Bulgarian",
|
| 14 |
+
"cat": "Catalan",
|
| 15 |
+
"ceb": "Cebuano",
|
| 16 |
+
"ces": "Czech",
|
| 17 |
+
"ckb": "Central Kurdish",
|
| 18 |
+
"cmn": "Mandarin Chinese",
|
| 19 |
+
"cym": "Welsh",
|
| 20 |
+
"dan": "Danish",
|
| 21 |
+
"deu": "German",
|
| 22 |
+
"ell": "Greek",
|
| 23 |
+
"eng": "English",
|
| 24 |
+
"est": "Estonian",
|
| 25 |
+
"eus": "Basque",
|
| 26 |
+
"fin": "Finnish",
|
| 27 |
+
"fra": "French",
|
| 28 |
+
"gaz": "West Central Oromo",
|
| 29 |
+
"gle": "Irish",
|
| 30 |
+
"glg": "Galician",
|
| 31 |
+
"guj": "Gujarati",
|
| 32 |
+
"heb": "Hebrew",
|
| 33 |
+
"hin": "Hindi",
|
| 34 |
+
"hrv": "Croatian",
|
| 35 |
+
"hun": "Hungarian",
|
| 36 |
+
"hye": "Armenian",
|
| 37 |
+
"ibo": "Igbo",
|
| 38 |
+
"ind": "Indonesian",
|
| 39 |
+
"isl": "Icelandic",
|
| 40 |
+
"ita": "Italian",
|
| 41 |
+
"jav": "Javanese",
|
| 42 |
+
"jpn": "Japanese",
|
| 43 |
+
"kam": "Kamba",
|
| 44 |
+
"kan": "Kannada",
|
| 45 |
+
"kat": "Georgian",
|
| 46 |
+
"kaz": "Kazakh",
|
| 47 |
+
"kea": "Kabuverdianu",
|
| 48 |
+
"khk": "Halh Mongolian",
|
| 49 |
+
"khm": "Khmer",
|
| 50 |
+
"kir": "Kyrgyz",
|
| 51 |
+
"kor": "Korean",
|
| 52 |
+
"lao": "Lao",
|
| 53 |
+
"lit": "Lithuanian",
|
| 54 |
+
"ltz": "Luxembourgish",
|
| 55 |
+
"lug": "Ganda",
|
| 56 |
+
"luo": "Luo",
|
| 57 |
+
"lvs": "Standard Latvian",
|
| 58 |
+
"mai": "Maithili",
|
| 59 |
+
"mal": "Malayalam",
|
| 60 |
+
"mar": "Marathi",
|
| 61 |
+
"mkd": "Macedonian",
|
| 62 |
+
"mlt": "Maltese",
|
| 63 |
+
"mni": "Meitei",
|
| 64 |
+
"mya": "Burmese",
|
| 65 |
+
"nld": "Dutch",
|
| 66 |
+
"nno": "Norwegian Nynorsk",
|
| 67 |
+
"nob": "Norwegian Bokm\u00e5l",
|
| 68 |
+
"npi": "Nepali",
|
| 69 |
+
"nya": "Nyanja",
|
| 70 |
+
"oci": "Occitan",
|
| 71 |
+
"ory": "Odia",
|
| 72 |
+
"pan": "Punjabi",
|
| 73 |
+
"pbt": "Southern Pashto",
|
| 74 |
+
"pes": "Western Persian",
|
| 75 |
+
"pol": "Polish",
|
| 76 |
+
"por": "Portuguese",
|
| 77 |
+
"ron": "Romanian",
|
| 78 |
+
"rus": "Russian",
|
| 79 |
+
"slk": "Slovak",
|
| 80 |
+
"slv": "Slovenian",
|
| 81 |
+
"sna": "Shona",
|
| 82 |
+
"snd": "Sindhi",
|
| 83 |
+
"som": "Somali",
|
| 84 |
+
"spa": "Spanish",
|
| 85 |
+
"srp": "Serbian",
|
| 86 |
+
"swe": "Swedish",
|
| 87 |
+
"swh": "Swahili",
|
| 88 |
+
"tam": "Tamil",
|
| 89 |
+
"tel": "Telugu",
|
| 90 |
+
"tgk": "Tajik",
|
| 91 |
+
"tgl": "Tagalog",
|
| 92 |
+
"tha": "Thai",
|
| 93 |
+
"tur": "Turkish",
|
| 94 |
+
"ukr": "Ukrainian",
|
| 95 |
+
"urd": "Urdu",
|
| 96 |
+
"uzn": "Northern Uzbek",
|
| 97 |
+
"vie": "Vietnamese",
|
| 98 |
+
"xho": "Xhosa",
|
| 99 |
+
"yor": "Yoruba",
|
| 100 |
+
"yue": "Cantonese",
|
| 101 |
+
"zlm": "Colloquial Malay",
|
| 102 |
+
"zsm": "Standard Malay",
|
| 103 |
+
"zul": "Zulu",
|
| 104 |
+
}
|