Update sync_streamlit_to_space.yml

README.md

@@ -1,33 +1,17 @@
 ---
 title: Zabanshenas
-emoji: 👁
-colorFrom: pink
-colorTo: pink
+emoji: 🕵
+colorFrom: blue
+colorTo: blue
 sdk: streamlit
 app_file: app.py
 pinned: false
 ---
 
-# Configuration
+# Zabanshenas
 
-`title`: _string_  
-Display title for the Space
+A Transformer-based solution for identifying the most likely language of a written document/text. 
+**Zabanshenas** is a Persian word that has two meanings:
 
-`emoji`: _string_  
-Space emoji (emoji-only character allowed)
-
-`colorFrom`: _string_  
-Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
-
-`colorTo`: _string_  
-Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
-
-`sdk`: _string_  
-Can be either `gradio` or `streamlit`
-
-`app_file`: _string_  
-Path to your main application file (which contains either `gradio` or `streamlit` Python code).  
-Path is relative to the root of the repository.
-
-`pinned`: _boolean_  
-Whether the Space stays on top of your list.
+- A person who studies linguistics.
+- A way to identify the type of written language.

app.py

@@ -0,0 +1,185 @@
+import streamlit as st
+
+from typing import Any, Dict, Optional
+import numpy as np
+import torch
+
+from transformers import AutoTokenizer
+from transformers import AutoModelForSequenceClassification
+
+from libs.normalizer import Normalizer
+from libs.languages import languages
+from libs.examples import EXAMPLES
+from libs.dummy import outputs as dummy_outputs
+from libs.utils import plot_result
+
+import meta
+
+
+class Zabanshenas:
+    def __init__(
+            self,
+            model_name_or_path: str = "m3hrdadfi/zabanshenas-roberta-base-mix",
+            by_gpu: bool = False
+    ) -> None:
+        self.debug = True
+        self.dummy_outputs = dummy_outputs
+        self.device = torch.device("cpu" if not by_gpu else "cuda")
+        self.model_name_or_path = model_name_or_path
+
+        self.tokenizer = None
+        self.model = None
+        self.normalizer = None
+        self.languages = None
+        self.framework = "pt"
+        self.max_length = 512
+        self.top_k = 5
+
+    def load(self):
+        if not self.debug:
+            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path)
+            self.model = AutoModelForSequenceClassification.from_pretrained(self.model_name_or_path).to(self.device)
+
+        self.normalizer = Normalizer()
+        self.languages = languages
+
+    def ensure_tensor_on_device(self, **inputs):
+        """
+        Ensure PyTorch tensors are on the specified device.
+        """
+
+        return {
+            name: tensor.to(self.device) if isinstance(tensor, torch.Tensor) else tensor
+            for name, tensor in inputs.items()
+        }
+
+    def _parse_and_tokenize(
+            self,
+            inputs,
+            do_normalization: bool = True,
+            max_length: int = 512,
+            padding: bool = True,
+            add_special_tokens: bool = True,
+            truncation: bool = True,
+    ):
+        """
+        Parse arguments and tokenize
+        """
+        inputs = [self.normalizer(item) for item in inputs]
+        max_length = min(max_length, self.max_length)
+        inputs = self.tokenizer(
+            inputs,
+            max_length=max_length,
+            add_special_tokens=add_special_tokens,
+            return_tensors=self.framework,
+            padding=padding,
+            truncation=truncation,
+        )
+
+        return inputs
+
+    def _forward(
+            self,
+            inputs,
+            return_tensors: bool = True
+    ):
+        with torch.no_grad():
+            inputs = self.ensure_tensor_on_device(**inputs)
+            predictions = self.model(**inputs)[0].cpu()
+
+        if return_tensors:
+            return predictions
+        else:
+            return predictions.numpy()
+
+    def detect(
+            self,
+            texts,
+            max_length: int = 128,
+            do_normalization: bool = True
+    ):
+        if self.debug:
+            return self.dummy_outputs
+
+        texts = [texts] if not isinstance(texts, list) else texts
+        inputs = self._parse_and_tokenize(texts, do_normalization=do_normalization, max_length=max_length)
+        outputs = self._forward(inputs, return_tensors=False)
+        scores = np.exp(outputs) / np.exp(outputs).sum(-1, keepdims=True)
+
+        results = [
+            [
+                {
+                    "language": self.languages.get(self.model.config.id2label[i], None),
+                    "code": self.model.config.id2label[i],
+                    "score": score.item()
+                } for i, score in enumerate(item)
+            ] for item in scores
+        ]
+        results = [list(sorted(result, key=lambda kv: kv["score"], reverse=True)) for result in results]
+
+        return results
+
+
+@st.cache(allow_output_mutation=True)
+def load_language_detector():
+    detector = Zabanshenas()
+    detector.load()
+    return detector
+
+
+def main():
+    st.set_page_config(
+        page_title="Zabanshenas",
+        page_icon="🕵",
+        layout="wide",
+        initial_sidebar_state="expanded"
+    )
+    detector = load_language_detector()
+
+    col1, col2 = st.beta_columns([6, 4])
+    with col2:
+        st.markdown(meta.INFO, unsafe_allow_html=True)
+
+    with col1:
+        prompts = list(EXAMPLES.keys()) + ["Custom"]
+        prompt = st.selectbox(
+            'Examples (select from this list)',
+            prompts,
+            # index=len(prompts) - 1,
+            index=0
+        )
+
+        if prompt == "Custom":
+            prompt_box = ""
+        else:
+            prompt_box = EXAMPLES[prompt]
+
+        text = st.text_area(
+            'Insert your text: ',
+            detector.normalizer(prompt_box),
+            height=200
+        )
+        text = detector.normalizer(text)
+        entered_text = st.empty()
+
+    detect_language = st.button('Detect Language !')
+
+    st.markdown(
+        "<hr />",
+        unsafe_allow_html=True
+    )
+    if detect_language:
+        words = text.split()
+        with st.spinner("Detecting..."):
+            if not len(words) > 3:
+                entered_text.markdown(
+                    "Insert your text (at least three words)"
+                )
+            else:
+                top_languages = detector.detect(text, max_length=min(len(words), detector.max_length))
+                top_languages = top_languages[0][:detector.top_k]
+                plot_result(top_languages)
+
+
+if __name__ == '__main__':
+    main()

libs/dummy.py

@@ -0,0 +1,1179 @@
+outputs = [
+    [
+        {
+            "language": "Persian",
+            "code": "fas",
+            "score": 0.6105580925941467
+        },
+        {
+            "language": "Gilaki",
+            "code": "glk",
+            "score": 0.29982829093933105
+        },
+        {
+            "language": "Northern Luri",
+            "code": "lrc",
+            "score": 0.04840774089097977
+        },
+        {
+            "language": "Mazanderani",
+            "code": "mzn",
+            "score": 0.030142733827233315
+        },
+        {
+            "language": "South Azerbaijani",
+            "code": "azb",
+            "score": 0.005220199003815651
+        },
+        {
+            "language": "Urdu",
+            "code": "urd",
+            "score": 0.0019745035097002983
+        },
+        {
+            "language": "Pushto",
+            "code": "pus",
+            "score": 0.0015690263826400042
+        },
+        {
+            "language": "Western Panjabi",
+            "code": "pnb",
+            "score": 0.0005721596535295248
+        },
+        {
+            "language": "Central Kurdish",
+            "code": "ckb",
+            "score": 0.00025537016335874796
+        },
+        {
+            "language": "Sindhi",
+            "code": "snd",
+            "score": 0.0001820324978325516
+        },
+        {
+            "language": "Egyptian Arabic",
+            "code": "arz",
+            "score": 0.0001247940381290391
+        },
+        {
+            "language": "Arabic",
+            "code": "ara",
+            "score": 7.754910620860755e-05
+        },
+        {
+            "language": "Korean",
+            "code": "kor",
+            "score": 5.718228203477338e-05
+        },
+        {
+            "language": "Fiji Hindi",
+            "code": "hif",
+            "score": 3.5903740354115143e-05
+        },
+        {
+            "language": "Uighur",
+            "code": "uig",
+            "score": 3.5565532016335055e-05
+        },
+        {
+            "language": "Maori",
+            "code": "mri",
+            "score": 2.1078320060041733e-05
+        },
+        {
+            "language": "Literary Chinese",
+            "code": "lzh",
+            "score": 2.09943773370469e-05
+        },
+        {
+            "language": "Navajo",
+            "code": "nav",
+            "score": 1.8877935872296803e-05
+        },
+        {
+            "language": "Mongolian",
+            "code": "mon",
+            "score": 1.783044899639208e-05
+        },
+        {
+            "language": "Basque",
+            "code": "eus",
+            "score": 1.2980432074982673e-05
+        },
+        {
+            "language": "Moksha",
+            "code": "mdf",
+            "score": 1.2325609532126691e-05
+        },
+        {
+            "language": "Tongan",
+            "code": "ton",
+            "score": 1.1610675755946431e-05
+        },
+        {
+            "language": "Min Dong",
+            "code": "cdo",
+            "score": 1.1508132956805639e-05
+        },
+        {
+            "language": "Sinhala",
+            "code": "sin",
+            "score": 1.0617596672091167e-05
+        },
+        {
+            "language": "Venetian",
+            "code": "vec",
+            "score": 1.0375520105299074e-05
+        },
+        {
+            "language": "Western Mari",
+            "code": "mrj",
+            "score": 1.0316403859178536e-05
+        },
+        {
+            "language": "Malayalam",
+            "code": "mal",
+            "score": 1.0265099263051525e-05
+        },
+        {
+            "language": "Interlingua",
+            "code": "ina",
+            "score": 1.0040446795755997e-05
+        },
+        {
+            "language": "Tatar",
+            "code": "tat",
+            "score": 9.836200661084149e-06
+        },
+        {
+            "language": "Cantonese",
+            "code": "zh-yue",
+            "score": 9.80662207439309e-06
+        },
+        {
+            "language": "Wu Chinese",
+            "code": "wuu",
+            "score": 9.661145668360405e-06
+        },
+        {
+            "language": "Igbo",
+            "code": "ibo",
+            "score": 9.207592484017368e-06
+        },
+        {
+            "language": "Waray",
+            "code": "war",
+            "score": 8.970115231932141e-06
+        },
+        {
+            "language": "Yiddish",
+            "code": "yid",
+            "score": 8.926748705562204e-06
+        },
+        {
+            "language": "Udmurt",
+            "code": "udm",
+            "score": 8.702583727426827e-06
+        },
+        {
+            "language": "Dhivehi",
+            "code": "div",
+            "score": 8.36203707876848e-06
+        },
+        {
+            "language": "Newari",
+            "code": "new",
+            "score": 8.140945283230394e-06
+        },
+        {
+            "language": "Karachay-Balkar",
+            "code": "krc",
+            "score": 8.123539373627864e-06
+        },
+        {
+            "language": "Lojban",
+            "code": "jbo",
+            "score": 8.114019692584407e-06
+        },
+        {
+            "language": "Sanskrit",
+            "code": "san",
+            "score": 8.087784408417065e-06
+        },
+        {
+            "language": "Luganda",
+            "code": "lug",
+            "score": 8.023569534998387e-06
+        },
+        {
+            "language": "Maithili",
+            "code": "mai",
+            "score": 7.723083399469033e-06
+        },
+        {
+            "language": "Kirghiz",
+            "code": "kir",
+            "score": 7.715119863860309e-06
+        },
+        {
+            "language": "Standard Chinese",
+            "code": "zho",
+            "score": 7.5126054071006365e-06
+        },
+        {
+            "language": "Amharic",
+            "code": "amh",
+            "score": 7.451813871739432e-06
+        },
+        {
+            "language": "Chechen",
+            "code": "che",
+            "score": 7.444541097356705e-06
+        },
+        {
+            "language": "Gujarati",
+            "code": "guj",
+            "score": 7.395997727144277e-06
+        },
+        {
+            "language": "Tibetan",
+            "code": "bod",
+            "score": 7.390805421891855e-06
+        },
+        {
+            "language": "Komi",
+            "code": "kom",
+            "score": 7.373077551164897e-06
+        },
+        {
+            "language": "Lao",
+            "code": "lao",
+            "score": 7.351867679972202e-06
+        },
+        {
+            "language": "Wolof",
+            "code": "wol",
+            "score": 7.305452982109273e-06
+        },
+        {
+            "language": "Silesian",
+            "code": "szl",
+            "score": 7.301976893359097e-06
+        },
+        {
+            "language": "Northern Sotho",
+            "code": "nso",
+            "score": 7.2927336987049785e-06
+        },
+        {
+            "language": "Armenian",
+            "code": "hye",
+            "score": 7.243447726068553e-06
+        },
+        {
+            "language": "Arpitan",
+            "code": "frp",
+            "score": 7.137540251278551e-06
+        },
+        {
+            "language": "Bishnupriya",
+            "code": "bpy",
+            "score": 7.062033091642661e-06
+        },
+        {
+            "language": "Azerbaijani",
+            "code": "aze",
+            "score": 6.906778253323864e-06
+        },
+        {
+            "language": "Tajik",
+            "code": "tgk",
+            "score": 6.730050699843559e-06
+        },
+        {
+            "language": "Old English ",
+            "code": "ang",
+            "score": 6.6442084971640725e-06
+        },
+        {
+            "language": "Marathi",
+            "code": "mar",
+            "score": 6.63194168737391e-06
+        },
+        {
+            "language": "Kurdish",
+            "code": "kur",
+            "score": 6.615779057028703e-06
+        },
+        {
+            "language": "Lithuanian",
+            "code": "lit",
+            "score": 6.561998816323467e-06
+        },
+        {
+            "language": "Russian",
+            "code": "rus",
+            "score": 6.4370215113740414e-06
+        },
+        {
+            "language": "Tulu",
+            "code": "tcy",
+            "score": 6.370255960064242e-06
+        },
+        {
+            "language": "Extremaduran",
+            "code": "ext",
+            "score": 6.3398160818906035e-06
+        },
+        {
+            "language": "Aymara",
+            "code": "aym",
+            "score": 6.288398708420573e-06
+        },
+        {
+            "language": "Lower Sorbian",
+            "code": "dsb",
+            "score": 6.209619641595054e-06
+        },
+        {
+            "language": "Classical Nahuatl",
+            "code": "nci",
+            "score": 5.954705557087436e-06
+        },
+        {
+            "language": "Polish",
+            "code": "pol",
+            "score": 5.952156243438367e-06
+        },
+        {
+            "language": "Cebuano",
+            "code": "ceb",
+            "score": 5.911888820264721e-06
+        },
+        {
+            "language": "Hakka Chinese",
+            "code": "hak",
+            "score": 5.756284735980444e-06
+        },
+        {
+            "language": "Georgian",
+            "code": "kat",
+            "score": 5.656391749653267e-06
+        },
+        {
+            "language": "Mingrelian",
+            "code": "xmf",
+            "score": 5.57373004994588e-06
+        },
+        {
+            "language": "Telugu",
+            "code": "tel",
+            "score": 5.5334053286060225e-06
+        },
+        {
+            "language": "Doteli",
+            "code": "dty",
+            "score": 5.510717073775595e-06
+        },
+        {
+            "language": "Portuguese",
+            "code": "por",
+            "score": 5.50901131646242e-06
+        },
+        {
+            "language": "Komi-Permyak",
+            "code": "koi",
+            "score": 5.447328476293478e-06
+        },
+        {
+            "language": "Eastern Mari",
+            "code": "mhr",
+            "score": 5.414771294454113e-06
+        },
+        {
+            "language": "Lezghian",
+            "code": "lez",
+            "score": 5.2741329454875086e-06
+        },
+        {
+            "language": "Nepali (macrolanguage)",
+            "code": "nep",
+            "score": 5.273408532957546e-06
+        },
+        {
+            "language": "Samogitian",
+            "code": "sgs",
+            "score": 5.207636149862083e-06
+        },
+        {
+            "language": "Bhojpuri",
+            "code": "bho",
+            "score": 5.19551804245566e-06
+        },
+        {
+            "language": "Occitan",
+            "code": "oci",
+            "score": 5.172901182959322e-06
+        },
+        {
+            "language": "Western Frisian",
+            "code": "fry",
+            "score": 5.066170615464216e-06
+        },
+        {
+            "language": "Vlaams",
+            "code": "vls",
+            "score": 5.014707312511746e-06
+        },
+        {
+            "language": "Japanese",
+            "code": "jpn",
+            "score": 4.986791282135528e-06
+        },
+        {
+            "language": "V\u00f5ro",
+            "code": "vro",
+            "score": 4.9785726332629565e-06
+        },
+        {
+            "language": "Rusyn",
+            "code": "rue",
+            "score": 4.937043286190601e-06
+        },
+        {
+            "language": "Hindi",
+            "code": "hin",
+            "score": 4.9325194595439825e-06
+        },
+        {
+            "language": "Sicilian",
+            "code": "scn",
+            "score": 4.8434171731059905e-06
+        },
+        {
+            "language": "Somali",
+            "code": "som",
+            "score": 4.722482117358595e-06
+        },
+        {
+            "language": "Galician",
+            "code": "glg",
+            "score": 4.664954758482054e-06
+        },
+        {
+            "language": "Kazakh",
+            "code": "kaz",
+            "score": 4.485120825847844e-06
+        },
+        {
+            "language": "Kannada",
+            "code": "kan",
+            "score": 4.438274572748924e-06
+        },
+        {
+            "language": "Oromo",
+            "code": "orm",
+            "score": 4.422903202794259e-06
+        },
+        {
+            "language": "Albanian",
+            "code": "sqi",
+            "score": 4.410150268085999e-06
+        },
+        {
+            "language": "Minangkabau",
+            "code": "min",
+            "score": 4.407007509144023e-06
+        },
+        {
+            "language": "Finnish",
+            "code": "fin",
+            "score": 4.374884611024754e-06
+        },
+        {
+            "language": "Ossetian",
+            "code": "oss",
+            "score": 4.322507265897002e-06
+        },
+        {
+            "language": "Volap\u00fck",
+            "code": "vol",
+            "score": 4.30220188718522e-06
+        },
+        {
+            "language": "Min Nan Chinese",
+            "code": "nan",
+            "score": 4.2357942220405675e-06
+        },
+        {
+            "language": "Bashkir",
+            "code": "bak",
+            "score": 4.212616204313235e-06
+        },
+        {
+            "language": "Ligurian",
+            "code": "lij",
+            "score": 4.1821313061518595e-06
+        },
+        {
+            "language": "Welsh",
+            "code": "cym",
+            "score": 4.174029982095817e-06
+        },
+        {
+            "language": "Slovene",
+            "code": "slv",
+            "score": 4.172954504610971e-06
+        },
+        {
+            "language": "Dimli",
+            "code": "diq",
+            "score": 4.078176516486565e-06
+        },
+        {
+            "language": "Chuvash",
+            "code": "chv",
+            "score": 4.048466053063748e-06
+        },
+        {
+            "language": "Panjabi",
+            "code": "pan",
+            "score": 3.940522674383828e-06
+        },
+        {
+            "language": "Cornish",
+            "code": "cor",
+            "score": 3.940297119697789e-06
+        },
+        {
+            "language": "West Low German",
+            "code": "nds-nl",
+            "score": 3.926987574232044e-06
+        },
+        {
+            "language": "Cherokee",
+            "code": "chr",
+            "score": 3.9112833292165305e-06
+        },
+        {
+            "language": "Ido",
+            "code": "ido",
+            "score": 3.892145286954474e-06
+        },
+        {
+            "language": "Friulian",
+            "code": "fur",
+            "score": 3.869370175380027e-06
+        },
+        {
+            "language": "Ukrainian",
+            "code": "ukr",
+            "score": 3.7814761526533403e-06
+        },
+        {
+            "language": "Vietnamese",
+            "code": "vie",
+            "score": 3.7795757634739857e-06
+        },
+        {
+            "language": "Emilian",
+            "code": "egl",
+            "score": 3.7286854421836324e-06
+        },
+        {
+            "language": "Hungarian",
+            "code": "hun",
+            "score": 3.706084498844575e-06
+        },
+        {
+            "language": "Haitian Creole",
+            "code": "hat",
+            "score": 3.6860656109638512e-06
+        },
+        {
+            "language": "Jamaican Patois",
+            "code": "jam",
+            "score": 3.6750652725459076e-06
+        },
+        {
+            "language": "Turkmen",
+            "code": "tuk",
+            "score": 3.6414037367649144e-06
+        },
+        {
+            "language": "Gagauz",
+            "code": "gag",
+            "score": 3.6310443647380453e-06
+        },
+        {
+            "language": "Yakut",
+            "code": "sah",
+            "score": 3.611620968513307e-06
+        },
+        {
+            "language": "Breton",
+            "code": "bre",
+            "score": 3.5204120649723336e-06
+        },
+        {
+            "language": "Afrikaans",
+            "code": "afr",
+            "score": 3.5164177916158224e-06
+        },
+        {
+            "language": "Assamese",
+            "code": "asm",
+            "score": 3.5076063795713708e-06
+        },
+        {
+            "language": "Crimean Tatar",
+            "code": "crh",
+            "score": 3.4974791560671292e-06
+        },
+        {
+            "language": "Tswana",
+            "code": "tsn",
+            "score": 3.4639840578165604e-06
+        },
+        {
+            "language": "Malagasy",
+            "code": "mlg",
+            "score": 3.4424308523739455e-06
+        },
+        {
+            "language": "Tamil",
+            "code": "tam",
+            "score": 3.433554866205668e-06
+        },
+        {
+            "language": "Belarusian (Taraschkewiza)",
+            "code": "be-tarask",
+            "score": 3.4065565159835387e-06
+        },
+        {
+            "language": "Scottish Gaelic",
+            "code": "gla",
+            "score": 3.383374632903724e-06
+        },
+        {
+            "language": "Latin",
+            "code": "lat",
+            "score": 3.299320724181598e-06
+        },
+        {
+            "language": "Chavacano",
+            "code": "cbk",
+            "score": 3.277132236689795e-06
+        },
+        {
+            "language": "Tarantino dialect",
+            "code": "roa-tara",
+            "score": 3.2704483601264656e-06
+        },
+        {
+            "language": "Modern Greek",
+            "code": "ell",
+            "score": 3.2669522624928504e-06
+        },
+        {
+            "language": "Ladino",
+            "code": "lad",
+            "score": 3.1890219815977616e-06
+        },
+        {
+            "language": "Latgalian",
+            "code": "ltg",
+            "score": 3.1830948046263075e-06
+        },
+        {
+            "language": "Pampanga",
+            "code": "pam",
+            "score": 3.1460281206818763e-06
+        },
+        {
+            "language": "Tagalog",
+            "code": "tgl",
+            "score": 3.100457433902193e-06
+        },
+        {
+            "language": "Hebrew",
+            "code": "heb",
+            "score": 3.0715009415871464e-06
+        },
+        {
+            "language": "Serbo-Croatian",
+            "code": "hbs",
+            "score": 3.050950908800587e-06
+        },
+        {
+            "language": "Achinese",
+            "code": "ace",
+            "score": 3.0138855890982086e-06
+        },
+        {
+            "language": "Italian",
+            "code": "ita",
+            "score": 3.003329993589432e-06
+        },
+        {
+            "language": "English",
+            "code": "eng",
+            "score": 2.97778979074792e-06
+        },
+        {
+            "language": "Burmese",
+            "code": "mya",
+            "score": 2.9546490623033606e-06
+        },
+        {
+            "language": "Spanish",
+            "code": "spa",
+            "score": 2.9272057417983888e-06
+        },
+        {
+            "language": "Papiamento",
+            "code": "pap",
+            "score": 2.8780641514458694e-06
+        },
+        {
+            "language": "Sardinian",
+            "code": "srd",
+            "score": 2.866505383281037e-06
+        },
+        {
+            "language": "Esperanto",
+            "code": "epo",
+            "score": 2.848199301297427e-06
+        },
+        {
+            "language": "Serbian",
+            "code": "srp",
+            "score": 2.7479175059852423e-06
+        },
+        {
+            "language": "Zeeuws",
+            "code": "zea",
+            "score": 2.7430314730736427e-06
+        },
+        {
+            "language": "Czech",
+            "code": "ces",
+            "score": 2.7409500944486354e-06
+        },
+        {
+            "language": "Bengali",
+            "code": "ben",
+            "score": 2.6958239232044434e-06
+        },
+        {
+            "language": "Erzya",
+            "code": "myv",
+            "score": 2.6273187359038275e-06
+        },
+        {
+            "language": "Croatian",
+            "code": "hrv",
+            "score": 2.6178654479735997e-06
+        },
+        {
+            "language": "Buryat",
+            "code": "bxr",
+            "score": 2.60430465459649e-06
+        },
+        {
+            "language": "Swahili (macrolanguage)",
+            "code": "swa",
+            "score": 2.6016373340098653e-06
+        },
+        {
+            "language": "Pangasinan",
+            "code": "pag",
+            "score": 2.60037768384791e-06
+        },
+        {
+            "language": "Xhosa",
+            "code": "xho",
+            "score": 2.580123918960453e-06
+        },
+        {
+            "language": "Bosnian",
+            "code": "bos",
+            "score": 2.5763115445442963e-06
+        },
+        {
+            "language": "Low German",
+            "code": "nds",
+            "score": 2.5743340756889665e-06
+        },
+        {
+            "language": "Kinyarwanda",
+            "code": "kin",
+            "score": 2.568235458966228e-06
+        },
+        {
+            "language": "Aromanian",
+            "code": "rup",
+            "score": 2.520287125662435e-06
+        },
+        {
+            "language": "Aragonese",
+            "code": "arg",
+            "score": 2.4836215288814856e-06
+        },
+        {
+            "language": "Tetum",
+            "code": "tet",
+            "score": 2.396502168267034e-06
+        },
+        {
+            "language": "Quechua",
+            "code": "que",
+            "score": 2.3799134396540467e-06
+        },
+        {
+            "language": "Livvi-Karelian",
+            "code": "olo",
+            "score": 2.3709426386631094e-06
+        },
+        {
+            "language": "Kashubian",
+            "code": "csb",
+            "score": 2.358733354412834e-06
+        },
+        {
+            "language": "Avar",
+            "code": "ava",
+            "score": 2.330698407604359e-06
+        },
+        {
+            "language": "Hausa",
+            "code": "hau",
+            "score": 2.286114295202424e-06
+        },
+        {
+            "language": "Ripuarisch",
+            "code": "ksh",
+            "score": 2.254129412904149e-06
+        },
+        {
+            "language": "Bulgarian",
+            "code": "bul",
+            "score": 2.2492179141408997e-06
+        },
+        {
+            "language": "Oriya",
+            "code": "ori",
+            "score": 2.1661755909008207e-06
+        },
+        {
+            "language": "Interlingue",
+            "code": "ile",
+            "score": 2.059975486190524e-06
+        },
+        {
+            "language": "Guarani",
+            "code": "grn",
+            "score": 2.024690957114217e-06
+        },
+        {
+            "language": "Banjar",
+            "code": "bjn",
+            "score": 2.0237362150510307e-06
+        },
+        {
+            "language": "Thai",
+            "code": "tha",
+            "score": 2.01868806470884e-06
+        },
+        {
+            "language": "Dutch",
+            "code": "nld",
+            "score": 1.9297158360132016e-06
+        },
+        {
+            "language": "Kabyle",
+            "code": "kab",
+            "score": 1.9132662600895856e-06
+        },
+        {
+            "language": "Palatine German",
+            "code": "pfl",
+            "score": 1.9122355752188014e-06
+        },
+        {
+            "language": "Javanese",
+            "code": "jav",
+            "score": 1.8900879013017402e-06
+        },
+        {
+            "language": "Banyumasan",
+            "code": "map-bms",
+            "score": 1.8552185565567925e-06
+        },
+        {
+            "language": "Faroese",
+            "code": "fao",
+            "score": 1.8414674514133367e-06
+        },
+        {
+            "language": "Scots",
+            "code": "sco",
+            "score": 1.818199393710529e-06
+        },
+        {
+            "language": "Central Khmer",
+            "code": "khm",
+            "score": 1.7993022538576042e-06
+        },
+        {
+            "language": "Slovak",
+            "code": "slk",
+            "score": 1.7988603531193803e-06
+        },
+        {
+            "language": "Belarusian",
+            "code": "bel",
+            "score": 1.782583581189101e-06
+        },
+        {
+            "language": "Swedish",
+            "code": "swe",
+            "score": 1.7702136574371252e-06
+        },
+        {
+            "language": "Saterfriesisch",
+            "code": "stq",
+            "score": 1.7663436437942437e-06
+        },
+        {
+            "language": "Latvian",
+            "code": "lav",
+            "score": 1.7178032294395962e-06
+        },
+        {
+            "language": "Konkani",
+            "code": "kok",
+            "score": 1.690383783170546e-06
+        },
+        {
+            "language": "Tuvan",
+            "code": "tyv",
+            "score": 1.672853159107035e-06
+        },
+        {
+            "language": "Walloon",
+            "code": "wln",
+            "score": 1.6722132158975e-06
+        },
+        {
+            "language": "Sranan",
+            "code": "srn",
+            "score": 1.646132773203135e-06
+        },
+        {
+            "language": "Picard",
+            "code": "pcd",
+            "score": 1.6385885146519286e-06
+        },
+        {
+            "language": "Limburgan",
+            "code": "lim",
+            "score": 1.6372666777897393e-06
+        },
+        {
+            "language": "French",
+            "code": "fra",
+            "score": 1.6239549722740776e-06
+        },
+        {
+            "language": "Icelandic",
+            "code": "isl",
+            "score": 1.5904075780781568e-06
+        },
+        {
+            "language": "Irish",
+            "code": "gle",
+            "score": 1.5750525790281245e-06
+        },
+        {
+            "language": "Corsican",
+            "code": "cos",
+            "score": 1.570832523611898e-06
+        },
+        {
+            "language": "Alemannic German",
+            "code": "als",
+            "score": 1.5651218063794659e-06
+        },
+        {
+            "language": "German",
+            "code": "deu",
+            "score": 1.5594737305946182e-06
+        },
+        {
+            "language": "Upper Sorbian",
+            "code": "hsb",
+            "score": 1.5125158370210556e-06
+        },
+        {
+            "language": "Romanian",
+            "code": "ron",
+            "score": 1.5119784393391456e-06
+        },
+        {
+            "language": "Manx",
+            "code": "glv",
+            "score": 1.5035052456369158e-06
+        },
+        {
+            "language": "Lingala",
+            "code": "lin",
+            "score": 1.493238073635439e-06
+        },
+        {
+            "language": "Malay",
+            "code": "msa",
+            "score": 1.4067626352698426e-06
+        },
+        {
+            "language": "Maltese",
+            "code": "mlt",
+            "score": 1.370485165352875e-06
+        },
+        {
+            "language": "Luxembourgish",
+            "code": "ltz",
+            "score": 1.3397349221122568e-06
+        },
+        {
+            "language": "Estonian",
+            "code": "est",
+            "score": 1.3280839539220324e-06
+        },
+        {
+            "language": "Kabardian",
+            "code": "kbd",
+            "score": 1.3062604011793155e-06
+        },
+        {
+            "language": "Macedonian",
+            "code": "mkd",
+            "score": 1.2802570381609257e-06
+        },
+        {
+            "language": "Pennsylvania German",
+            "code": "pdc",
+            "score": 1.2550040082714986e-06
+        },
+        {
+            "language": "Sundanese",
+            "code": "sun",
+            "score": 1.1068191270169336e-06
+        },
+        {
+            "language": "Iloko",
+            "code": "ilo",
+            "score": 1.0791690101541462e-06
+        },
+        {
+            "language": "Karakalpak",
+            "code": "kaa",
+            "score": 1.0603262126096524e-06
+        },
+        {
+            "language": "Norwegian Nynorsk",
+            "code": "nno",
+            "score": 1.0554679192864569e-06
+        },
+        {
+            "language": "Yoruba",
+            "code": "yor",
+            "score": 1.046297711582156e-06
+        },
+        {
+            "language": "Neapolitan",
+            "code": "nap",
+            "score": 1.0279602520313347e-06
+        },
+        {
+            "language": "Danish",
+            "code": "dan",
+            "score": 1.0038916116172913e-06
+        },
+        {
+            "language": "Indonesian",
+            "code": "ind",
+            "score": 9.83746303973021e-07
+        },
+        {
+            "language": "Mirandese",
+            "code": "mwl",
+            "score": 8.806521236692788e-07
+        },
+        {
+            "language": "Catalan",
+            "code": "cat",
+            "score": 8.687447348165733e-07
+        },
+        {
+            "language": "Turkish",
+            "code": "tur",
+            "score": 8.384120064874878e-07
+        },
+        {
+            "language": "Veps",
+            "code": "vep",
+            "score": 7.812500371073838e-07
+        },
+        {
+            "language": "Bokm\u00e5l",
+            "code": "nob",
+            "score": 7.427178161378833e-07
+        },
+        {
+            "language": "Shona",
+            "code": "sna",
+            "score": 6.660703775196453e-07
+        },
+        {
+            "language": "Bavarian",
+            "code": "bar",
+            "score": 6.222485353646334e-07
+        },
+        {
+            "language": "Uzbek",
+            "code": "uzb",
+            "score": 6.021850822435226e-07
+        },
+        {
+            "language": "Central Bikol",
+            "code": "bcl",
+            "score": 5.77034370508045e-07
+        },
+        {
+            "language": "Asturian",
+            "code": "ast",
+            "score": 5.743918336520437e-07
+        },
+        {
+            "language": "Lombard",
+            "code": "lmo",
+            "score": 4.6301857992148143e-07
+        },
+        {
+            "language": "Romansh",
+            "code": "roh",
+            "score": 4.5534079617937095e-07
+        },
+        {
+            "language": "Narom",
+            "code": "nrm",
+            "score": 3.6611126574825903e-07
+        },
+        {
+            "language": "Northern Sami",
+            "code": "sme",
+            "score": 1.0723972820869676e-07
+        }
+    ]
+]

libs/examples.py

@@ -0,0 +1,40 @@
+EXAMPLES = {
+    'Example 1 - Paragraph - Wu Chinese': '里维拉军政府 ( 西班牙文 Directorio militar de Primo de Rivera 是20世纪20年代 西班牙军事人物大里维拉立在西班牙国王阿方索十三世搭西班牙天主教个支持下建立个威权主义政府 迭个政府自称超出传统个政治党派 一切为国王搭西班牙个稳定服务 伊拉个支持者是爱国同盟 弗过 因为遇到经济大萧条 威权政府立在1930年垮台 之后西班牙拿国王废脱 成立西班牙第二共和国',
+    'Example 2 - Paragraph - Scots': 'Polyandry ( frae Greek : πολυ - poly - , " " mony " " and ἀνήρ anēr , " " man " " ) involves mairiage that includes mair nor twa pairtners an can faw unner the broader category o polyamory . Mair speceefically , it is a form o polygamy , whaur a woman takes twa or mair husbands at the same time . Polyandry is contrasted wi polygyny , involvin ane male an twa or mair females . If a mairiage involves a plural nummer o " " husbands an wives " " pairteecipants o each gender , then it can be cried polyamory , group or conjynt mairiage . In its broadest uise , polyandry refers tae sexual relations wi multiple males within or withoot mairiage .',
+    'Example 3 - Paragraph - Sinhala': 'ට්රූඑක්ස් ක් රියාවලියට විකල්පයක් ලෙස මැලොන්ඩයමයිඩ් ( malondiamide ) යොදා ගන්නා නිස්සාරණ ක් රමයක් හදුන්වා දී ඇත . ඩයමෙක්ස් ( DIAMEX ; DIAMide Extraction ) ක් රියාවලියේ වාසියක් වන්නේ කාබන් , හයිඩ් රජන් , නයිට් රජන් හා ඔක්සිජන් හැර වෙනත් මූල ද් රව් ය අඩංගු කාබනික අපද් රව් ය උත්පාදනය වීම වැළකීමයි . මෙම අපද් රව් ය අම්ල වැසිවලට උරදෙන ආම්ලික වායු නොසෑදෙන සේ දහනය කළ හැකිය . ඩයමෙක්ස් ක් රමය ප් රංශ CEA මගින් යුරෝපයේ භාවිතයට ගනී . ක් රියාවලිය ගැන දැනට පවතින දැනුමෙන් කාර්මික කම්හල් ඉදි කිරීමට තරම් මෙම ක් රියාවලිය පරිණත වී ඇත . මෙම ක් රියාවලිය ද භාවිතා කරන්නේ ද් රව් යතා යාන්ත් රණයක්ය .',
+    'Example 4 - Paragraph - Asturian': "Presupuestos públicos pa la igualdá y el desenvolvimientu sustentable para con ello tresformar l ' actual modelu de desenvolvimientu atendiendo les causes estructurales que xeneren y reproducen desigualdaes por ello participa nel ambito internacional y rexonal nel siguimientu de los Oxetivos de Desenvolvimientu del Mileniu ( ODS ) y agora na axenda 2030 .",
+    'Example 5 - Paragraph - Swahili (macrolanguage)': 'Kuna nyota nyingi katika eneo la kundinyota hii lakini zote si angavu sana . Nyota angavu zaidi ni Beta Aquarii ambayo ni nyota jitu kubwa njano mwenye uangavu unaoonekana wa 2 . 9 .',
+    'Example 6 - Paragraph - Czech': 'CASSE , Gilbert ; CUNDALL , Peter ; TULLY , Anthony . IJN ARGENTINA MARU : Tabular Record of Movement [ online ] . combinedfleet . com , 2015 - 09 - 13 , [ cit . 2015 - 09 - 13 ] . Dostupné online . ( anglicky )',
+    'Example 7 - Paragraph - Lingala': 'Mayanzi ekutanaka mingi mingi na Afrika , na amérika mpé na bisika ya moyi makasi . Liyanzi elingaka bisika ya zélo mpé ya salité mpo na ko kota na nzoto ya nyama mosusu to na nzoto ya moto .',
+    'Example 8 - Paragraph - Thai': 'โดยที่ an > 0 สำหรับทุก n แต่ละพจน์ของอนุกรมจะมีเครื่องหมายบวกและลบสลับกัน เช่นเดียวกับอนุกรมอื่นๆ อนุกรมสลับจะลู่เข้าก็ต่อเมื่อลำดับของผลบวกจำกัดพจน์ลู่เข้า',
+    'Example 9 - Paragraph - Waray': 'An Orphnus rufithorax in uska species han Coleoptera nga ginhulagway ni Benderitter hadton 1914 . An Orphnus rufithorax in nahilalakip ha genus nga Orphnus , ngan familia nga Orphnidae . Waray hini subspecies nga nakalista .',
+    'Example 10 - Paragraph - Gagauz': 'Ama buna bakmadaan , masmediya milionnarca insannarın ölmesinnän korkudêr . Sansın büün medişina XIX - cu üzyılın uurunda bulunarmış gibi . Düünnedä panika başlamaya yakın .',
+    'Example 11 - Paragraph - Western Panjabi': 'پینڈولم دی فزکس سب توں پہلا گلیلیو نیں 1602 دی چ سمجی سی تے اس ویلے توں 1930 تک پینڈولم نوں ویلے دے سب توں ٹھیک حساب لئی ورتیا جاندا ��ی پینڈولم نوں ویلا نپن دے علاوہ زمین دی کچھ نپن تے بھونچال دے زور نپن لئی وی ورتیا جاندا سی',
+    'Example 12 - Paragraph - West Low German': 'Hoog Buurlo is n gehucht in de gemeente Apeldoorne , in de Nederlandse provinsie Gelderland . t Ligt ten westen van de stad Apeldoorne en iets ten oosten van Radio Kootwiek .',
+    'Example 13 - Paragraph - Quechua': '| 5 ñiqin pachakwata | 6 ñiqin pachakwata | 7 ñiqin pachakwata | | 510 watakuna | 520 watakuna | 530 watakuna | 540 watakuna | 550 watakuna | 560 watakuna | 570 watakuna | | | 545 | 546 | 547 | 548 | 549 | 550 | 551 | 552 | 553 | |',
+    'Example 14 - Paragraph - Serbo-Croatian': 'Sočanica je naselje u opštini Leposavić na Kosovu i Metohiji . Površina katastarske opštine Sočanica gde je atar naselja iznosi 1 . 929 ha . Sedište je mesne zajednice Sočanica . Naselje Sočanica poslednjih godina prerasta iz seoskog naselja u varošicu , nalazi se 5 km južno od Leposavića sa desne strane reke Ibar . Srednja nadmorska visina naselja iznosi 636m . U pisanim izvorima selo se prvi put pominje 1315 . godine , u povelji srpskog kralja Stefana Milutina manastiru Banjskoj .',
+    'Example 15 - Paragraph - Tongan': 'ʻOku pehē ʻe Niel Gunson naʻe ʻafifio ʻa e Tuʻi Tonga ʻi he taimi ko ia ʻi Manuʻa pea naʻa nau hoko ki he Tuʻi Manuʻa . Ko ia ai ʻoku hala ha tuʻi ʻi Tongatapu , pea naʻe kamata ha holongā tuʻi foʻou .',
+    'Example 16 - Paragraph - Rusyn': 'Прыпять - є єднов з найдовшых рік на Україні і Европы . Тече на теріторії Білорусії і Україны . Довжка рікы є 775 км . Коло міста Чорнобыль ся вливать до водозбіру рікы Днїпр і так там кінчыть свою путь .',
+    'Example 17 - Paragraph - Ladino': 'Ciruelos de Cervera es un puevlo de la Provinsia de Burgos en la junta de Kastiya i Leon en Espanya . Tiene una povlasion de 111 avitants i una ekstension de 37 , 874 km² ( 2015 ) .',
+    'Example 18 - Paragraph - Bosnian': 'Po posljednjem službenom popisu stanovništva iz 1991 . godine , općina Hadžići ( u to vrijeme jedna od 5 prigradskih općina Grada Sarajeva ) imala je 24 . 200 stanovnika , raspoređenih u 62 naselja .',
+    'Example 19 - Paragraph - Chuvash': 'Мăн Агыйдел ( Агыйдел ) Раççей территоринчи юханшыв . Вологда облаçĕ , Киров облаçĕ , Коми Республики территорипе юхать . Шарженг юханшывăн сылтăм çыранĕпе 143 км вăрринчен юханшыва юхса кĕрет . Юханшыв тăршшĕ 10 км .',
+    'Example 20 - Paragraph - Dhivehi': 'ގުރުދާ ބަލި ބޮޑުވަމުންދާވަރަކަށް ލޭގައި ޖަމާވަމުންދާ ބޭކާރު މާއްދާތައް ހަށިގަނޑުން ބޭރުކުރުމަށް ހަށިގަނޑު ނުކުޅެދެއެވެ . ނަތީޖާއެއްގެ ގޮތުގައި މިމާއްދާތައް ގިނަވެ ވިހަވާ މިންވަރަށް އިތުރުވެއެވެ . ލޭގެ Pްރެޝަރ އިތުރުވެފައިވާނަމަ މިމައްސަލަ އިތުރަށް ގޯސްވެއެވެ . ގުރުދާ ބަލިން ރައްކާތެރިވެ ވީހާވެސް ކުރިން ސިއްހީ ފަރުވާ ފެށުމަށްޓަކައި ބަލީގެ ކުރީކޮޅުގައި ބަލި ފާހަގަކުރުމަށް މަސައްކަތް ކުރުމަކީ މުހިއްމު ކަމެކެވެ .',
+    'Example 21 - Sentence - Western Panjabi': 'یکی از این آبخورگاه ها در ضلع شرقی صحن و در مقابل مقبره راجه قرار داشته و بهره هند ( طایفه ای از اسماعیلیان ) آن را نوسازی کرده بودند و در جوار آن نیز دو درخت میوه و یک درخت سدر بوده است .',
+    'Example 22 - Sentence - Tamil': 'மேற்குறிப்பிட்ட கட்சிகளைத் தவிர முஸ்லிம் லீக் , இந்திய கம்யூனிஸ்ட் கட்சி , ஃபார்வார்டு ப்ளாக் , சி .',
+    'Example 23 - Sentence - Basque': 'Hala ere , garai horretan ELAk ez zituen langile etorkinak onartzen , afiliatzeko lehen lau abizenetatik bat gutxienez euskal jatorrikoa izatea eskatzen baitzuen oraindik .',
+    'Example 24 - Sentence - Livvi-Karelian': 'Niilöis lapset lujendetah omua tervehytty , harjavutah vedeh .',
+    'Example 25 - Sentence - Eastern Mari': '3 : Тÿнямбал да руш классике , рушлаш кусарыме сборник - влак .',
+    'Example 26 - Sentence - Breton': "Brudet eo bet e Breizh abalamour d ' e enebiezh ruz ouzh Diwan hag ouzh ar brezhoneg .",
+    'Example 27 - Sentence - Bosnian': 'Sa 52 godine vratio se u Veneciju i ponudio svoje usluge svojim donedavnim progoniteljima za koje je radio kao špijun i za to su ga plaćali .',
+    'Example 28 - Sentence - Uzbek': 'diapazoni N .', 'Example 29 - Sentence - Esperanto': 'Loa !',
+    'Example 30 - Sentence - Lezghian': 'Кабир - Казмаляр ) Дагъустан республикадин Мегьарамдхуьруьн районда авай , « КьепIир Къазмайрин » хуьруьнсоветдик акатзавай хуьр .',
+    'Example 31 - Sentence - Norwegian Nynorsk': 'Morfaren var elles ein velkjend filosof og samfunnskritikar , Aleksandr Radisjtsjev .',
+    'Example 32 - Sentence - Papiamento': 'Esaki tabata inaceptabel pa e politiconan di Curaçao .',
+    'Example 33 - Sentence - Azerbaijani': 'Müharibədən evə sağ - salamat dönən Xındı Məmməd öz gözəl sənətini davam etdirmişdir .',
+    'Example 34 - Sentence - Volapük': 'Lödanadensit äbinon mö mens 460 , 2 a km² .',
+    'Example 35 - Sentence - Konkani': '13 No creature is concealed from him , but everything is naked and exposed to the eyes of him to whom we must render an account',
+    'Example 36 - Sentence - Latgalian': '1990 godā solu reorganizej kai pogostu .',
+    'Example 37 - Sentence - Swedish': 'Inga underarter finns listade i Catalogue of Life .',
+    'Example 38 - Sentence - Tuvan': 'Ол ам оралакчы сайыт апарган кижи чүге дуза кадып шыдавас деп бодап , эжинге бүзүрел Серге - Байырны ооӊ - биле ужуражылгаже эккээр .',
+    'Example 39 - Sentence - Malagasy': "200 no isan ' ny kisoa .",
+    'Example 40 - Sentence - English': 'The convention followed after a request by the Bulgarian government on 24 September asking for a ceasefire .'}

libs/languages.py

@@ -0,0 +1,237 @@
+languages = {
+    "ace": "Achinese",
+    "afr": "Afrikaans",
+    "als": "Alemannic German",
+    "amh": "Amharic",
+    "ang": "Old English ",
+    "ara": "Arabic",
+    "arg": "Aragonese",
+    "arz": "Egyptian Arabic",
+    "asm": "Assamese",
+    "ast": "Asturian",
+    "ava": "Avar",
+    "aym": "Aymara",
+    "azb": "South Azerbaijani",
+    "aze": "Azerbaijani",
+    "bak": "Bashkir",
+    "bar": "Bavarian",
+    "bcl": "Central Bikol",
+    "be-tarask": "Belarusian (Taraschkewiza)",
+    "bel": "Belarusian",
+    "ben": "Bengali",
+    "bho": "Bhojpuri",
+    "bjn": "Banjar",
+    "bod": "Tibetan",
+    "bos": "Bosnian",
+    "bpy": "Bishnupriya",
+    "bre": "Breton",
+    "bul": "Bulgarian",
+    "bxr": "Buryat",
+    "cat": "Catalan",
+    "cbk": "Chavacano",
+    "cdo": "Min Dong",
+    "ceb": "Cebuano",
+    "ces": "Czech",
+    "che": "Chechen",
+    "chr": "Cherokee",
+    "chv": "Chuvash",
+    "ckb": "Central Kurdish",
+    "cor": "Cornish",
+    "cos": "Corsican",
+    "crh": "Crimean Tatar",
+    "csb": "Kashubian",
+    "cym": "Welsh",
+    "dan": "Danish",
+    "deu": "German",
+    "diq": "Dimli",
+    "div": "Dhivehi",
+    "dsb": "Lower Sorbian",
+    "dty": "Doteli",
+    "egl": "Emilian",
+    "ell": "Modern Greek",
+    "eng": "English",
+    "epo": "Esperanto",
+    "est": "Estonian",
+    "eus": "Basque",
+    "ext": "Extremaduran",
+    "fao": "Faroese",
+    "fas": "Persian",
+    "fin": "Finnish",
+    "fra": "French",
+    "frp": "Arpitan",
+    "fry": "Western Frisian",
+    "fur": "Friulian",
+    "gag": "Gagauz",
+    "gla": "Scottish Gaelic",
+    "gle": "Irish",
+    "glg": "Galician",
+    "glk": "Gilaki",
+    "glv": "Manx",
+    "grn": "Guarani",
+    "guj": "Gujarati",
+    "hak": "Hakka Chinese",
+    "hat": "Haitian Creole",
+    "hau": "Hausa",
+    "hbs": "Serbo-Croatian",
+    "heb": "Hebrew",
+    "hif": "Fiji Hindi",
+    "hin": "Hindi",
+    "hrv": "Croatian",
+    "hsb": "Upper Sorbian",
+    "hun": "Hungarian",
+    "hye": "Armenian",
+    "ibo": "Igbo",
+    "ido": "Ido",
+    "ile": "Interlingue",
+    "ilo": "Iloko",
+    "ina": "Interlingua",
+    "ind": "Indonesian",
+    "isl": "Icelandic",
+    "ita": "Italian",
+    "jam": "Jamaican Patois",
+    "jav": "Javanese",
+    "jbo": "Lojban",
+    "jpn": "Japanese",
+    "kaa": "Karakalpak",
+    "kab": "Kabyle",
+    "kan": "Kannada",
+    "kat": "Georgian",
+    "kaz": "Kazakh",
+    "kbd": "Kabardian",
+    "khm": "Central Khmer",
+    "kin": "Kinyarwanda",
+    "kir": "Kirghiz",
+    "koi": "Komi-Permyak",
+    "kok": "Konkani",
+    "kom": "Komi",
+    "kor": "Korean",
+    "krc": "Karachay-Balkar",
+    "ksh": "Ripuarisch",
+    "kur": "Kurdish",
+    "lad": "Ladino",
+    "lao": "Lao",
+    "lat": "Latin",
+    "lav": "Latvian",
+    "lez": "Lezghian",
+    "lij": "Ligurian",
+    "lim": "Limburgan",
+    "lin": "Lingala",
+    "lit": "Lithuanian",
+    "lmo": "Lombard",
+    "lrc": "Northern Luri",
+    "ltg": "Latgalian",
+    "ltz": "Luxembourgish",
+    "lug": "Luganda",
+    "lzh": "Literary Chinese",
+    "mai": "Maithili",
+    "mal": "Malayalam",
+    "map-bms": "Banyumasan",
+    "mar": "Marathi",
+    "mdf": "Moksha",
+    "mhr": "Eastern Mari",
+    "min": "Minangkabau",
+    "mkd": "Macedonian",
+    "mlg": "Malagasy",
+    "mlt": "Maltese",
+    "mon": "Mongolian",
+    "mri": "Maori",
+    "mrj": "Western Mari",
+    "msa": "Malay",
+    "mwl": "Mirandese",
+    "mya": "Burmese",
+    "myv": "Erzya",
+    "mzn": "Mazanderani",
+    "nan": "Min Nan Chinese",
+    "nap": "Neapolitan",
+    "nav": "Navajo",
+    "nci": "Classical Nahuatl",
+    "nds": "Low German",
+    "nds-nl": "West Low German",
+    "nep": "Nepali (macrolanguage)",
+    "new": "Newari",
+    "nld": "Dutch",
+    "nno": "Norwegian Nynorsk",
+    "nob": "Bokmål",
+    "nrm": "Narom",
+    "nso": "Northern Sotho",
+    "oci": "Occitan",
+    "olo": "Livvi-Karelian",
+    "ori": "Oriya",
+    "orm": "Oromo",
+    "oss": "Ossetian",
+    "pag": "Pangasinan",
+    "pam": "Pampanga",
+    "pan": "Panjabi",
+    "pap": "Papiamento",
+    "pcd": "Picard",
+    "pdc": "Pennsylvania German",
+    "pfl": "Palatine German",
+    "pnb": "Western Panjabi",
+    "pol": "Polish",
+    "por": "Portuguese",
+    "pus": "Pushto",
+    "que": "Quechua",
+    "roa-tara": "Tarantino dialect",
+    "roh": "Romansh",
+    "ron": "Romanian",
+    "rue": "Rusyn",
+    "rup": "Aromanian",
+    "rus": "Russian",
+    "sah": "Yakut",
+    "san": "Sanskrit",
+    "scn": "Sicilian",
+    "sco": "Scots",
+    "sgs": "Samogitian",
+    "sin": "Sinhala",
+    "slk": "Slovak",
+    "slv": "Slovene",
+    "sme": "Northern Sami",
+    "sna": "Shona",
+    "snd": "Sindhi",
+    "som": "Somali",
+    "spa": "Spanish",
+    "sqi": "Albanian",
+    "srd": "Sardinian",
+    "srn": "Sranan",
+    "srp": "Serbian",
+    "stq": "Saterfriesisch",
+    "sun": "Sundanese",
+    "swa": "Swahili (macrolanguage)",
+    "swe": "Swedish",
+    "szl": "Silesian",
+    "tam": "Tamil",
+    "tat": "Tatar",
+    "tcy": "Tulu",
+    "tel": "Telugu",
+    "tet": "Tetum",
+    "tgk": "Tajik",
+    "tgl": "Tagalog",
+    "tha": "Thai",
+    "ton": "Tongan",
+    "tsn": "Tswana",
+    "tuk": "Turkmen",
+    "tur": "Turkish",
+    "tyv": "Tuvan",
+    "udm": "Udmurt",
+    "uig": "Uighur",
+    "ukr": "Ukrainian",
+    "urd": "Urdu",
+    "uzb": "Uzbek",
+    "vec": "Venetian",
+    "vep": "Veps",
+    "vie": "Vietnamese",
+    "vls": "Vlaams",
+    "vol": "Volapük",
+    "vro": "Võro",
+    "war": "Waray",
+    "wln": "Walloon",
+    "wol": "Wolof",
+    "wuu": "Wu Chinese",
+    "xho": "Xhosa",
+    "xmf": "Mingrelian",
+    "yid": "Yiddish",
+    "yor": "Yoruba",
+    "zea": "Zeeuws",
+    "zh-yue": "Cantonese",
+    "zho": "Standard Chinese",
+}

libs/normalizer.py

@@ -0,0 +1,86 @@
+import re
+import regex
+import sys
+import textwrap
+from typing import Any, Dict, Optional
+
+punctuations = [
+    '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '.',
+    '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_',
+    '`', '{', '|', '}', '~', '»', '«', '“', '”', "-",
+]
+
+
+class Normalizer:
+    """A general normalizer for every language"""
+
+    _whitelist = r"[" + "\p{N}\p{L}\p{M}" + re.escape("".join(punctuations)) + "]+"
+    _dictionary = {}
+
+    def __init__(
+            self,
+            whitelist: str = None,
+            dictionary: Dict[str, str] = None,
+    ) -> None:
+        self.whitelist = whitelist if whitelist and isinstance(whitelist, str) else self._whitelist
+        self.dictionary = dictionary if dictionary and isinstance(dictionary, dict) else self._dictionary
+
+    def chars_to_map(self, sentence: str) -> str:
+        """Maps every character, words, and phrase into a proper one.
+
+        Args:
+            sentence (str): A piece of text.
+        """
+        if not len(self.dictionary) > 0:
+            return sentence
+
+        pattern = "|".join(map(re.escape, self.dictionary.keys()))
+        return re.sub(pattern, lambda m: self.dictionary[m.group()], str(sentence))
+
+    def chars_to_preserve(
+            self,
+            sentence: str,
+    ) -> str:
+        """Keeps specified characters from sentence
+
+        Args:
+            sentence (str): A piece of text.
+        """
+        try:
+            tokenized = regex.findall(self.whitelist, sentence)
+            return " ".join(tokenized)
+        except Exception as error:
+            print(
+                textwrap.dedent(
+                    f"""
+                    Bad characters range {self.whitelist},
+                    {error}
+                    """
+                )
+            )
+            raise
+
+    def text_level_normalizer(self, text: str) -> str:
+        """A text level of normalization"""
+
+        text = regex.sub(r"([" + re.escape("".join(punctuations)) + "])", r" \1 ", text)
+        text = text.strip()
+
+        return text
+
+    def __call__(
+            self,
+            text: str,
+            do_lowercase: Optional[bool] = False
+    ) -> Any:
+        """Normalization caller"""
+
+        text = self.chars_to_map(text)
+        text = self.chars_to_preserve(text)
+        text = self.text_level_normalizer(text)
+        text = re.sub(r"\s+", " ", text)
+
+        if do_lowercase:
+            text = text.lower()
+
+        return text

libs/utils.py

@@ -0,0 +1,25 @@
+import streamlit as st
+import numpy as np
+import plotly.express as px
+
+
+def plot_result(top_languages):
+    top_k = len(top_languages)
+    languages = [f'{r["language"]} ({r["code"]})' for r in top_languages]
+    scores = np.array([r["score"] for r in top_languages])
+    scores *= 100
+    fig = px.bar(
+        x=scores,
+        y=languages,
+        orientation='h',
+        labels={'x': 'Confidence', 'y': 'Language'},
+        text=scores,
+        range_x=(0, 115),
+        title=f'Top Detections {top_k}',
+        color=np.linspace(0, 1, len(scores)),
+        color_continuous_scale='GnBu'
+    )
+    fig.update(layout_coloraxis_showscale=False)
+    fig.update_traces(texttemplate='%{text:0.1f}%', textposition='outside')
+    fig.update_layout(yaxis={'categoryorder': 'total ascending'})
+    st.plotly_chart(fig, use_container_width=True)

meta.py

@@ -0,0 +1,8 @@
+INFO = """
+# Zabanshenas 🕵
+
+A Transformer-based solution for identifying the most likely language of a written document/text. Zabanshenas is a Persian word that has two meanings:
+
+- A person who studies linguistics.
+- A way to identify the type of written language.
+""".strip()

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+transformers
+torch
+regex
+plotly