7 languages supported

app.py

@@ -909,7 +909,7 @@ param_visu_langs = {
         "path_sentencepiece_model": f"./{lang_dataset_id}.sp.model",
         "path_kenlm_model": f"./{lang_dataset_id}.arpa.bin",
     }
-    for lang_dataset_id in ["en", "zh"]
+    for lang_dataset_id in ["eu", "ca", "zh", "en", "fr", "id", "es"]
 }
 
 visualization = Visualization(path_instructions, param_visu_langs)

ca.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ece1e503d4b44409069ea9c5c5125b74792b575143169e08cf9a27248f9a78e
+size 2809368958

ca.sp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abc6936e2ff5dcdc86962ffaeef48ef66f567d568ef7090d28123ed6618b455c
+size 946977

ca_examples_with_stats.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4207b45aa366ece2763a06565fcb771b86e433f2a6190248017f97e7534fa4a
+size 103605036

en_examples_with_stats.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d64ccdaa835843aabbcc7d74789140e5b68313dfff6c23f79a60b65d668633e8
-size 276101129
+oid sha256:1dccf03710e9dc7ec68c676175e711be815bc29a50260f5d334156b03fe2e6d1
+size 241408394

es.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26964ff8185eb105021fc0e9eaa0a1de590c4a12f8aa3fe12112b29d42281cf3
+size 3828418653

es.sp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aae545566a995d3374fbc8ac1d4e0c7073008da8ae32acfe7f176136a8efcf37
+size 961535

es_examples_with_stats.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d52760c4c961ebfe419a603a6d837619ca146656f563f5abbd140dec8fbe28e
+size 148378888

eu.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d04c4d1233b40044e2facc978987ecd4a6d4f84032f2af3f85f7079676fa08b
+size 774011873

eu.sp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:447cbd1714e51e6a7b4dd8ff55b7bd975fdb7f6ba873cb6f8a1fe36b5867dbb6
+size 955869

eu_examples_with_stats.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10a06ac7ed9b4c444f35fb9a3e3636a22689c198a6bdd4fd358b0eec50aa924d
+size 66358003

filtering.py

@@ -423,33 +423,91 @@ class Filtering:
         return cond
 
     @staticmethod
-    def compute_repetitions_ratio(document, repetitions_length):
-        def get_freq_ngrams(document, n):
-            ngrams = [document[i : i + n] for i in range(len(document) - n + 1)]
-            freq_ngrams = {}
-            for ngram in ngrams:
-                freq_ngrams[ngram] = freq_ngrams.get(ngram, 0) + 1
-            return freq_ngrams
-
-        freq_ngrams = get_freq_ngrams(document, repetitions_length)
-        if len(freq_ngrams) == 0:
+    def compute_character_repetition_ratio(document, character_repetition_length):
+        def get_freq_character_ngrams(document, n):
+            character_ngrams = [
+                document[i : i + n] for i in range(len(document) - n + 1)
+            ]
+            freq_character_ngrams = {}
+            for character_ngram in character_ngrams:
+                freq_character_ngrams[character_ngram] = (
+                    freq_character_ngrams.get(character_ngram, 0) + 1
+                )
+            return freq_character_ngrams
+
+        freq_character_ngrams = get_freq_character_ngrams(
+            document, character_repetition_length
+        )
+        if len(freq_character_ngrams) == 0:
+            return 0
+        freq_character_ngrams = list(freq_character_ngrams.values())
+        freq_character_ngrams = sorted(freq_character_ngrams, reverse=True)
+        val_less_than_one = len([el for el in freq_character_ngrams if el > 1])
+        num_rep_character_ngrams = min(
+            int(np.sqrt(len(freq_character_ngrams))),
+            len(freq_character_ngrams) - val_less_than_one,
+        )
+        character_repetition_ratio = sum(
+            freq_character_ngrams[:num_rep_character_ngrams]
+        ) / sum(freq_character_ngrams)
+        return character_repetition_ratio
+
+    @staticmethod
+    def check_character_repetition_removal(
+        document,
+        character_repetition_length,
+        character_repetition_max_cutoff,
+    ):
+        character_repetition_ratio = Filtering.compute_character_repetition_ratio(
+            document, character_repetition_length
+        )
+        cond = character_repetition_ratio <= character_repetition_max_cutoff
+        return cond
+
+    @staticmethod
+    def compute_word_repetition_ratio(
+        document, sentencepiece_model_tok, strip_characters, word_repetition_length
+    ):
+        def get_freq_word_ngrams(
+            document, sentencepiece_model_tok, strip_characters, n
+        ):
+            words = ModifyingDocuments.get_words_from_document(
+                document,
+                sentencepiece_model_tok,
+                lower_case=True,
+                strip_characters=strip_characters,
+            )
+            word_ngrams = [
+                " ".join(words[i : i + n]) for i in range(len(words) - n + 1)
+            ]
+            freq_word_ngrams = {}
+            for word_ngram in word_ngrams:
+                freq_word_ngrams[word_ngram] = freq_word_ngrams.get(word_ngram, 0) + 1
+            return freq_word_ngrams
+
+        freq_word_ngrams = get_freq_word_ngrams(
+            document, sentencepiece_model_tok, strip_characters, word_repetition_length
+        )
+        if len(freq_word_ngrams) == 0:
             return 0
-        freq_ngrams = list(freq_ngrams.values())
-        freq_ngrams = sorted(freq_ngrams, reverse=True)
-        num_rep_ngrams = int(np.sqrt(len(freq_ngrams)))
-        repetitions_ratio = sum(freq_ngrams[:num_rep_ngrams]) / sum(freq_ngrams)
-        return repetitions_ratio
+        freq_word_ngrams = list(freq_word_ngrams.values())
+        word_repetition_ratio = sum(
+            freq for freq in freq_word_ngrams if freq > 1
+        ) / sum(freq_word_ngrams)
+        return word_repetition_ratio
 
     @staticmethod
-    def check_repetitions_removal(
+    def check_word_repetition_removal(
         document,
-        repetitions_length,
-        repetitions_max_cutoff,
+        sentencepiece_model_tok,
+        strip_characters,
+        word_repetition_length,
+        word_repetition_max_cutoff,
     ):
-        repetitions_ratio = Filtering.compute_repetitions_ratio(
-            document, repetitions_length
+        word_repetition_ratio = Filtering.compute_word_repetition_ratio(
+            document, sentencepiece_model_tok, strip_characters, word_repetition_length
         )
-        cond = repetitions_ratio <= repetitions_max_cutoff
+        cond = word_repetition_ratio <= word_repetition_max_cutoff
         return cond
 
     @staticmethod
@@ -629,7 +687,7 @@ class Filtering:
             document=document,
             remove_non_printing_characters=True,
             strip=True,
-            lower_case=True,
+            lower_case=False,
             uniform_whitespace=True,
             replace_digits_with_zeros=True,
             replace_unicode_punctuation=True,
@@ -670,9 +728,12 @@ class Filtering:
         strip_characters,
         number_words_min_cutoff,
         number_words_max_cutoff,
-        cond_check_repetitions_removal,
-        repetitions_length,
-        repetitions_max_cutoff,
+        cond_check_character_repetition_removal,
+        character_repetition_length,
+        character_repetition_max_cutoff,
+        cond_check_word_repetition_removal,
+        word_repetition_length,
+        word_repetition_max_cutoff,
         cond_check_special_characters,
         special_characters,
         special_characters_max_cutoff,
@@ -703,11 +764,20 @@ class Filtering:
                 number_words_max_cutoff,
             ):
                 return False
-        if cond_check_repetitions_removal:
-            if not Filtering.check_repetitions_removal(
+        if cond_check_character_repetition_removal:
+            if not Filtering.check_character_repetition_removal(
                 document,
-                repetitions_length,
-                repetitions_max_cutoff,
+                character_repetition_length,
+                character_repetition_max_cutoff,
+            ):
+                return False
+        if cond_check_word_repetition_removal:
+            if not Filtering.check_word_repetition_removal(
+                document,
+                sentencepiece_model_tok,
+                strip_characters,
+                word_repetition_length,
+                word_repetition_max_cutoff,
             ):
                 return False
         if cond_check_special_characters:
@@ -797,9 +867,18 @@ class FunctionDatasetFiltering:
             strip_characters=self.param["strip_characters"],
             number_words_min_cutoff=self.param["number_words_min_cutoff"],
             number_words_max_cutoff=self.param["number_words_max_cutoff"],
-            cond_check_repetitions_removal=self.param["check_repetitions_removal"],
-            repetitions_length=self.param["repetitions_length"],
-            repetitions_max_cutoff=self.param["repetitions_max_cutoff"],
+            cond_check_character_repetition_removal=self.param[
+                "cond_check_character_repetition_removal"
+            ],
+            character_repetition_length=self.param["character_repetition_length"],
+            character_repetition_max_cutoff=self.param[
+                "character_repetition_max_cutoff"
+            ],
+            cond_check_word_repetition_removal=self.param[
+                "cond_check_word_repetition_removal"
+            ],
+            word_repetition_length=self.param["word_repetition_length"],
+            word_repetition_max_cutoff=self.param["word_repetition_max_cutoff"],
             cond_check_special_characters=self.param["cond_check_special_characters"],
             special_characters=self.param["special_characters"],
             special_characters_max_cutoff=self.param["special_characters_max_cutoff"],
@@ -855,10 +934,10 @@ class DatasetFiltering:
         self.path_dir_save_dataset = path_dir_save_dataset
 
     def modifying_documents(self):
-        dataset_modifying_documents = FunctionDatasetModifyingDocuments(
+        func_dataset_modifying_documents = FunctionDatasetModifyingDocuments(
             self.lang_dataset_id
         )
-        self.ds = self.ds.map(dataset_modifying_documents, num_proc=self.num_proc)
+        self.ds = self.ds.map(func_dataset_modifying_documents, num_proc=self.num_proc)
 
     def filtering(self):
         func_dataset_filtering = FunctionDatasetFiltering(

flagged_words.py

@@ -81,989 +81,281 @@ english_flagged_words = [
 flagged_words = {
     "ar": english_flagged_words
     + [
+        "إباحي",
         "احتلام",
+        "است",
+        "استمناء",
         "اغتصاب",
+        "أورغازم",
+        "إيروتيك",
+        "أير",
         "بز",
         "بزاز",
         "بظر",
+        "بورن",
         "بيضان",
-        "تمص",
+        "مص",
         "ثدي",
         "جماع",
+        "جنس",
         "حلمة",
+        "خلاعة",
         "خنثي",
         "خول",
+        "دعارة",
         "زب",
         "سحاق",
         "سحاقية",
         "سكس",
+        "سيكس",
         "شاذ",
+        "شبق",
         "شرج",
         "شرموطة",
+        "شهواني",
         "شهوة",
         "طيز",
+        "عادة السرية",
         "عاهرة",
         "عرص",
+        "فاسقة",
         "فرج",
         "قحبة",
         "قضيب",
         "كس",
-        "لبوة",
         "لحس",
         "لعق",
         "لواط",
         "لوطي",
-        "مبادل",
+        "مؤخرة",
         "متناك",
         "متناكة",
+        "مومس",
+        "مثير",
         "مص",
+        "مضاجعة",
         "مفلقسة",
+        "مني",
+        "مهتاج",
+        "نشوة",
+        "نكاح",
         "نيك",
     ],
     "ca": english_flagged_words
     + [
-        "anal",
-        "anus",
-        "cul",
-        "ass-fucker",
-        "asss",
-        "asshole",
-        "assholes",
-        "bolera",
-        "boles",
-        "bastardo",
-        "bellend",
-        "bestial",
-        "bestialitat",
-        "puta",
-        "femelles",
-        "picant",
-        "sagnant",
-        "mamada",
-        "bollok",
-        "boob",
-        "pits",
-        "buceta",
-        "bum",
-        "culata",
-        "catifa muncher",
-        "picar",
-        "cipa",
-        "clitoris",
-        "polla",
-        "galletejador",
-        "gallines",
-        "coon",
-        "merda",
-        "cum",
-        "correguda",
-        "cunillingus",
-        "boig",
-        "maleït",
+        "cagarro",
+        "cagarros",
+        "cipote",
+        "cipotes",
+        "collons",
+        "colló",
         "consolador",
         "consoladors",
-        "dink",
-        "canalla",
-        "duche",
-        "dique",
-        "ejaculació",
-        "ejaculat",
-        "ejacula",
-        "ejaculant",
-        "fag",
-        "fagging",
-        "fagot",
-        "fagots",
-        "fanny",
-        "felching",
-        "fel.lació",
-        "brida",
-        "follar",
-        "follat",
-        "escuradents",
-        "follant",
-        "folles",
-        "fucks",
-        "empacadora de llaminadures",
-        "déu maldit",
-        "deu meu",
-        "infern",
-        "hore",
-        "córrer",
-        "retrocés",
-        "kock",
-        "llavis",
-        "lujuria",
-        "lució",
-        "masoquista",
-        "masturbarse",
-        "puta mare",
-        "nazi",
-        "nigger",
-        "negres",
-        "orgasim",
+        "cony",
+        "conys",
+        "corre's",
+        "corre't",
+        "corregut",
+        "cunillingus",
+        "córrer-se",
+        "escorreguda",
+        "escorregudes",
+        "escorregut",
+        "escrot",
+        "escrots",
+        "escórre's",
+        "escórre't",
+        "escórrer-se",
+        "mamada",
+        "mamadera",
+        "mamaderes",
+        "mamades",
+        "masturba",
+        "masturbacions",
+        "masturbació",
+        "masturbant",
+        "masturbar",
+        "masturbar-se",
+        "masturbat",
+        "masturbats",
+        "masturbes",
         "orgasme",
         "orgasmes",
-        "pecker",
-        "penis",
-        "piss",
-        "mossegat",
-        "pisser",
-        "pisses",
-        "pissing",
-        "treure de polleguera",
-        "caca",
-        "porno",
+        "ou",
+        "ous",
+        "palla",
+        "palles",
         "pornografia",
-        "picades",
-        "pube",
-        "coques",
-        "gatet",
-        "violació",
-        "violador",
-        "recte",
-        "retard",
-        "rimming",
-        "sàdic",
-        "cargolar",
-        "escrot",
         "semen",
-        "sexe",
-        "shag",
-        "borratxos",
-        "transsexual",
-        "mossegar",
-        "shitted",
-        "skank",
-        "smegma",
-        "smut",
-        "arrebat",
-        "fill de puta",
-        "spac",
-        "spunk",
-        "testicle",
-        "tit",
-        "tetas",
-        "titt",
-        "turd",
-        "vagina",
-        "viagra",
-        "vulva",
-        "wang",
-        "wank",
-        "x classificat",
+        "semens",
+        "verga",
+        "vergues",
         "xxx",
     ],
     "en": english_flagged_words,
     "es": english_flagged_words
     + [
-        "Asesinato",
-        "Bollera",
-        "Cabrón",
-        "Caca",
-        "Chupada",
-        "Chupapollas",
-        "Chupetón",
-        "Concha de tu madre",
-        "Coprofagía",
-        "Coño",
-        "Culo",
-        "Drogas",
-        "Esperma",
-        "Fiesta de salchichas",
-        "Follador",
-        "Follar",
-        "Gilipichis",
-        "Gilipollas",
-        "Hacer una paja",
-        "Haciendo el amor",
-        "Heroína",
-        "Hija de puta",
-        "Hijaputa",
-        "Hijo de puta",
-        "Hijoputa",
-        "Idiota",
-        "Imbécil",
-        "Jilipollas",
-        "Kapullo",
-        "Lameculos",
-        "Maciza",
-        "Macizorra",
-        "Mamada",
-        "Marica",
-        "Mariconazo",
-        "Maricón",
-        "Mierda",
-        "Nazi",
-        "Orina",
-        "Pedo",
-        "Pendejo",
-        "Pervertido",
-        "Pezón",
-        "Pinche",
-        "Pis",
-        "Prostituta",
-        "Puta",
-        "Racista",
-        "Ramera",
-        "Semen",
-        "Sexo",
-        "Sexo oral",
-        "Soplagaitas",
-        "Soplapollas",
-        "Sádico",
-        "Tetas grandes",
-        "Travesti",
-        "Trio",
-        "Tía buena",
-        "Verga",
-        "Vulva",
-        "aborto",
-        "agallas",
-        "anal",
-        "ano",
-        "arrebatar",
-        "asno",
-        "atornillar",
-        "bastardo",
-        "bestial",
-        "bestialidad",
-        "bolas",
-        "bollok",
-        "bolsa de pelota",
-        "brida",
-        "buceta",
-        "cabron",
-        "cagadas",
-        "cagado",
-        "cagando",
-        "campana",
-        "carajo",
+        "chupar el coño",
+        "chupar la concha",
         "chupar la polla",
-        "cipa",
-        "clítoris",
-        "concha",
-        "consolador",
-        "consoladores",
-        "corrida",
-        "coño",
-        "coños",
-        "culo",
-        "culos",
+        "chupar la verga",
+        "comer el coño",
+        "comer la concha",
+        "comer la polla",
+        "comer la verga",
+        "coprofagía",
+        "correrse",
         "cunillingus",
-        "córneo",
-        "de mierda",
-        "dique",
-        "duche",
-        "enojado",
-        "escroto",
-        "espacio",
-        "estúpido",
-        "extremo",
-        "eyacula",
-        "eyaculación",
-        "eyaculado",
-        "eyacular",
         "fagging",
         "felación",
         "felching",
-        "folla",
         "follada",
         "follador de culo",
         "folladores",
-        "follar",
         "fudge packer",
-        "gallos",
-        "grieta",
+        "hacer una paja",
         "hacerse una paja",
-        "hijo de puta",
         "hore",
-        "infierno",
         "kock",
-        "labios vaginales",
-        "los pechos",
-        "lujuria",
+        "macizorra",
         "madre folladora",
-        "maldita sea",
-        "maldito",
-        "maldito sea",
         "mamada",
-        "mapache",
-        "maricones",
-        "maricón",
-        "martillo",
-        "masoquista",
-        "masturbarse",
-        "mear",
-        "mierda",
-        "molesto",
-        "muncher alfombra",
-        "nazi",
-        "negro",
-        "niggers",
-        "orgasimo",
-        "orgasmo",
-        "orgasmos",
-        "orinando",
-        "pelusa",
-        "pene",
-        "perra",
-        "perras",
         "perro follador",
-        "pinchazo",
-        "pinchazos",
         "pisser",
-        "polla",
-        "porno",
         "pornografía",
-        "pube",
-        "puta",
-        "putas",
-        "pájaro carpintero",
-        "quejas",
-        "recto",
-        "retardar",
-        "rimming",
-        "sangriento",
-        "semen",
-        "sexo",
+        "sado",
+        "sadomasoquismo",
+        "sadomasoquista",
+        "sexo anal",
         "skank",
         "smegma",
-        "sádico",
-        "testículo",
-        "teta",
-        "tetas",
-        "tirón",
-        "tizón",
-        "tonto",
-        "vagina",
-        "vete a la mierda",
-        "viagra",
-        "violación",
-        "violador",
-        "vulva",
-        "wang",
         "x clasificado",
-        "xxx",
-        "zurullo",
-    ],
-    "eu": english_flagged_words
-    + [
-        "abortu",
-        "anal",
-        "ipurdi",
-        "kabroi",
-        "puta",
-        "clitoris",
-        "cunillingus",
-        "madarikatu",
-        "zakil",
-        "hazia isuri",
-        "arraio",
-        "izorratu",
-        "infernu",
-        "emagaldu",
-        "lizunkeri",
-        "lizun",
-        "masokista",
-        "masturbatu",
-        "nazi",
-        "beltz",
-        "orgasmo",
-        "pixa",
-        "porno",
-        "pornografia",
-        "alu",
-        "bortxaketa",
-        "bortxatzaile",
-        "sadista",
-        "ipurzulo",
-        "hazi",
-        "semen",
-        "sexu",
-        "kaka",
-        "putaseme",
-        "barrabil",
-        "titi",
-        "bagina",
-        "viagra",
     ],
+    "eu": english_flagged_words + [],
     "fr": english_flagged_words
     + [
-        "MALPT",
-        "anal",
-        "anus",
-        "arracher",
-        "baise",
-        "baiser",
-        "baiseur de chien",
         "baiseurs",
-        "baisée",
-        "bander",
-        "bellend",
-        "bestial",
-        "bestialité",
-        "bigornette",
-        "bite",
-        "bitte",
-        "bloblos",
-        "bollok",
-        "boob",
-        "bordel",
-        "bourré",
-        "bourrée",
-        "bout",
-        "brackmard",
-        "branlage",
-        "branler",
+        "baiseur",
+        "baiseuse",
+        "baiseuses",
         "branlette",
-        "branleur",
+        "branlettes",
         "branleuse",
-        "bride",
-        "brouter le cresson",
-        "buceta",
-        "caca",
-        "chatte",
-        "chattes",
-        "chiasse",
-        "chienne",
-        "chiennes",
-        "chier",
-        "chiottes",
-        "chié",
-        "cipa",
-        "clito",
-        "clitoris",
-        "clochard",
-        "cochonneries",
-        "con",
-        "connard",
-        "connards",
-        "connasse",
-        "conne",
-        "convoitise",
-        "coq",
-        "coqs",
-        "corné",
-        "couilles",
-        "cramouille",
-        "cran",
-        "cul",
-        "culs",
+        "branleuses",
         "cunillingus",
-        "damné",
-        "des balles",
-        "digue",
-        "duché",
-        "déconne",
-        "déconner",
-        "emballeur de fudge",
-        "emmerdant",
-        "emmerder",
-        "emmerdeur",
-        "emmerdeuse",
-        "enculer",
-        "enculeur",
-        "enculeurs",
-        "enculé",
+        "cunilingus",
         "enculée",
-        "enfer",
-        "enfoiré",
-        "enfoirée",
-        "espacer",
-        "fagging",
-        "fagot",
-        "fagots",
-        "faire chier",
+        "enculées",
+        "enculation",
+        "enculations",
+        "enculement",
+        "enculements",
         "fellation",
-        "fente",
-        "fille de pute",
-        "fils de pute",
-        "folle",
-        "foutre",
-        "fuckings",
-        "gerbe",
-        "gerber",
-        "godemiché",
-        "godes",
-        "gouine",
-        "grande folle",
-        "grogniasse",
-        "gueule",
-        "hore",
-        "jouir",
-        "kock",
-        "la putain de ta mère",
-        "les lèvres",
-        "les seins",
-        "luxure",
-        "masochiste",
-        "masturber",
-        "merde",
-        "merdeuse",
-        "merdeux",
-        "merdique",
-        "meuf",
-        "mère enculée",
-        "ménage à trois",
-        "mésange",
-        "nazi",
-        "negro",
-        "nique ta mère",
-        "nique ta race",
-        "nègre",
-        "nègres",
-        "orgasim",
-        "orgasme",
-        "orgasmes",
-        "palucher",
-        "penchant",
-        "pipe",
-        "pipi",
-        "piquer",
-        "piqûres",
-        "pisse",
-        "pisser",
+        "fellations",
         "porno",
+        "pornos",
         "pornographie",
-        "pouffiasse",
-        "pousse-crotte",
-        "pube",
-        "putain",
-        "putain de",
-        "pute",
-        "pédale",
-        "pédé",
-        "pénis",
-        "péter",
-        "queue",
-        "quéquette",
-        "ramoner",
-        "rectum",
-        "retard",
-        "rimming",
-        "râpé",
-        "sac de billes",
-        "sac à foutre",
-        "sac à merde",
-        "sadique",
-        "salaud",
+        "pornographique",
+        "pornographiques",
         "salope",
         "salopes",
-        "sanglant",
-        "scrotum",
-        "se branler",
-        "seins",
-        "sexe",
-        "skank",
-        "smegma",
-        "sperme",
-        "suce",
         "suceuse",
-        "tanche",
-        "tapette",
-        "tapis muncher",
-        "testicule",
-        "teuch",
-        "titt",
-        "tremper",
-        "tringler",
-        "trique",
-        "troncher",
-        "trou du cul",
-        "turlute",
-        "vagin",
-        "viagra",
-        "violeur",
-        "vulve",
-        "wang",
-        "x évalué",
+        "suceuses",
         "xxx",
-        "zigounette",
-        "zizi",
-        "zut",
-        "éjaculant",
-        "éjaculation",
-        "éjacule",
-        "éjaculer",
-        "éjaculé",
-        "étron",
+    ],
+    "ha": english_flagged_words
+    + [
+        "bf",
+        "bura",
+        "burar",
+        "cin gindi",
+        "duri",
+        "durin",
+        "gindi",
+        "gindin",
+        "guntsu",
+        "guntsun",
+        "karuwa",
+        "karuwai",
+        "karuwar",
+        "maniyyi",
+        "maɗigo",
+        "maɗugo",
+        "nonuwa",
+        "shan gindi",
+        "tsuliya",
+        "tsuliyariskanci",
+        "ɗuwaiwai",
+        "ɗuwaiwan",
+        "ɗuwawu",
+        "ɗuwawun",
     ],
     "hi": english_flagged_words
     + [
-        "aand",
-        "aandu",
-        "balatkar",
-        "balatkari",
-        "behen chod",
-        "beti chod",
-        "bhadva",
-        "bhadve",
-        "bhandve",
-        "bhangi",
-        "bhootni ke",
-        "bhosad",
-        "bhosadi ke",
-        "bitching",
-        "blowjob",
-        "bollok",
-        "boobe",
-        "buceta",
-        "chakke",
-        "chinaal",
-        "chinki",
-        "chod",
-        "chodu",
-        "chodu bhagat",
-        "chooche",
-        "choochi",
-        "choope",
-        "choot",
-        "choot ke baal",
-        "chootia",
-        "chootiya",
-        "chuche",
-        "chuchi",
-        "chudaap",
-        "chudai khanaa",
-        "chudam chudai",
-        "chude",
-        "chut",
-        "chut ka chuha",
-        "chut ka churan",
-        "chut ka mail",
-        "chut ke baal",
-        "chut ke dhakkan",
-        "chut maarli",
-        "chutad",
-        "chutadd",
-        "chutan",
-        "chutia",
-        "chutiya",
-        "cipa",
-        "cunillingus",
-        "dink",
-        "duche",
-        "ejaculated",
-        "ejaculates",
-        "ejaculating",
-        "fagging",
-        "fagots",
-        "felching",
-        "fuckings",
-        "fucks",
-        "gaand",
-        "gaandfat",
-        "gaandmasti",
-        "gaandufad",
-        "gandfattu",
-        "gandu",
-        "gashti",
-        "gasti",
-        "ghassa",
-        "ghasti",
-        "gucchi",
-        "gucchu",
-        "harami",
-        "haramzade",
-        "hawas",
-        "hawas ke pujari",
-        "hijda",
-        "hijra",
-        "jhant",
-        "jhant chaatu",
-        "jhant ka keeda",
-        "jhant ke baal",
-        "jhant ke pissu",
-        "jhantu",
-        "kamine",
-        "kaminey",
-        "kanjar",
-        "kutta",
-        "kutta kamina",
-        "kutte ki aulad",
-        "kutte ki jat",
-        "kuttiya",
-        "loda",
-        "lodu",
-        "lund",
-        "lund choos",
-        "lund ka bakkal",
-        "lund khajoor",
-        "lundtopi",
-        "lundure",
-        "lusting",
-        "maa ki chut",
-        "maal",
-        "madar chod",
-        "madarchod",
-        "madhavchod",
-        "masochist",
-        "mooh mein le",
-        "mutth",
-        "mutthal",
-        "najayaz",
-        "najayaz aulaad",
-        "najayaz paidaish",
-        "orgasim",
-        "paki",
-        "pataka",
-        "patakha",
-        "pisser",
-        "pisses",
-        "pissing",
-        "pube",
-        "pussies",
-        "raand",
-        "randaap",
-        "randi",
-        "randi rona",
-        "rimming",
-        "saala",
-        "saala kutta",
-        "saali kutti",
-        "saali randi",
-        "shagging",
-        "shite",
-        "shitted",
-        "shitting",
-        "shitty",
-        "skank",
-        "sluts",
-        "spac",
-        "suar",
-        "suar ke lund",
-        "suar ki aulad",
-        "tatte",
-        "tatti",
-        "teri maa ka bhosada",
-        "teri maa ka boba chusu",
-        "teri maa ki behenchod ",
-        "teri maa ki chut",
-        "tharak",
-        "tharki",
-        "titt",
-        "tu chuda",
-        "turd",
-        "wank",
-        "xxx",
         "अंडकोश की थैली",
-        "अंडा",
-        "अरे नहीं",
-        "अश्लील",
-        "उल्लू",
         "एक्स रेटेड",
         "ओगाज़्म",
-        "कमबख्त",
-        "काम करना",
         "कामोद्दीपक चित्र",
         "कालीन का चूरा",
-        "किन्नर",
-        "कुतिया",
-        "कुत्ते-कमीने",
         "कून",
         "कॉक",
-        "गड़बड़",
-        "गधा कमीने",
-        "गधे",
-        "गर्भपात",
-        "गुदा",
         "गेंद का थैला",
-        "गेंदों",
-        "गोली चलाने की आवाज़",
-        "घटिया इंसान",
         "चाकलेट का रंग",
-        "चिंक",
-        "चुभन",
         "चूची",
         "चूतड़",
-        "चोंच",
-        "छीनना",
-        "जी में आये करो",
         "झटका बंद",
         "ठगना पैकर",
         "डिल्डो",
-        "दुष्ट",
-        "दूर जाने का अभद्र संकेत देना",
-        "धत् तेरे की",
-        "नरक",
-        "नाजी",
-        "निकला हुआ किनारा",
         "नितंब",
-        "पंगा लेना",
         "पिछाड़ी",
         "पीड़न कामुक",
-        "पेशाब",
         "पॉर्न",
         "फटना",
         "फूहड़",
-        "बकवास",
         "बट",
-        "बलात्कार",
         "बहुत मदहोश",
-        "बांध",
-        "बिल्ली",
         "बेल अंत",
-        "बेवकूफों",
-        "बोल पड़ना",
         "भगवान-शापित",
         "भगशेफ",
-        "मल",
-        "मलाशय",
         "माँ कमीने",
         "मुखमैथुन",
-        "मुर्गा",
-        "मुर्गा के",
         "मुर्गा चूसने वाला",
-        "मूर्ख",
-        "मैल",
-        "योनि",
-        "योनी",
-        "यौन-संबंध",
         "रक्तरंजित",
-        "लानत है",
-        "लिंग",
-        "लुटेरा",
         "लेबिया",
         "वहशी",
         "वहशीता",
-        "वियाग्रा",
-        "वीर्य",
-        "वेश्या",
         "वैंग",
-        "वो साले",
-        "शिफ़्ट को",
         "शिश्नमल",
         "संभोग सुख",
-        "सह",
         "सह शॉट",
-        "साहस",
-        "सिगरेट",
         "सींग का बना हुआ",
-        "स्तन",
-        "स्तनों",
-        "हवस",
-        "हस्तमैथुन",
-        "होमोसेक्सुअल",
         "होर",
+        "घपा घप",
+        "चुदाई",
+        "चुदक्कड़",
     ],
     "id": english_flagged_words
     + [
-        "abortus",
-        "anal",
-        "dubur",
-        "pantat",
-        "bajingan",
-        "keledai",
-        "keparat",
-        "tas bola",
-        "bola",
-        "bellend",
-        "kejam",
-        "kebinatangan",
-        "menggerutu",
-        "pelacur",
-        "berdarah",
-        "blowjob",
-        "bollok",
-        "dada",
-        "payudara",
-        "buceta",
-        "gelandangan",
-        "pengunyah karpet",
-        "celah",
-        "cipa",
-        "kelentit",
-        "kokang",
-        "pengisap ayam",
-        "ayam",
-        "coon",
-        "sampah",
-        "air mani",
-        "cumshot",
-        "cunillingus",
-        "vagina",
-        "mengutuk",
+        "bokep",
+        "coli",
+        "colmek",
+        "grepe",
+        "horni",
+        "janda",
+        "jembut",
+        "jilat memek",
+        "jilmek",
         "kontol",
-        "dildo",
-        "dink",
-        "anjing-keparat",
-        "duche",
-        "tanggul",
-        "berejakulasi",
-        "ejakulasi",
-        "homo",
-        "fagging",
-        "kayu bakar",
-        "penggemar",
-        "felching",
-        "fellatio",
-        "flens",
-        "brengsek",
-        "kacau",
-        "sialan",
-        "persetan",
-        "pengepakan fudge",
-        "terkutuk",
-        "ya tuhan",
-        "neraka",
-        "hore",
-        "terangsang",
-        "kock",
-        "labia",
-        "nafsu",
-        "bernafsu",
-        "masokis",
         "masturbasi",
-        "keparat ibu",
-        "nazi",
-        "orang negro",
-        "negro",
-        "orgasim",
-        "orgasme",
-        "cotok",
-        "penis",
-        "kencing",
-        "kesal",
-        "pisser",
-        "bikin",
-        "buritan",
-        "porno",
+        "memek",
+        "ngentot",
+        "ngewe",
+        "peju",
+        "pepek",
         "pornografi",
-        "tusukan",
-        "menusuk",
-        "pube",
-        "pussies",
-        "memperkosa",
-        "pemerkosa",
-        "memperlambat",
-        "rimming",
-        "sadis",
-        "meniduri",
-        "skrotum",
-        "seks",
-        "bercinta",
-        "waria",
-        "kotoran",
-        "shite",
-        "kengerian",
-        "dikirim",
-        "buang hajat",
-        "menyebalkan",
-        "smegma",
-        "jelaga",
-        "merebut",
-        "dasar bajingan",
-        "ruang",
-        "keberanian",
-        "buah pelir",
-        "titt",
-        "viagra",
-        "vulva",
-        "wang",
-        "terima kasih",
-        "x diberi peringkat",
-        "xxx",
+        "sange",
+        "sepong",
+        "tusbol",
     ],
     "kn": english_flagged_words
     + [
@@ -1682,6 +974,19 @@ flagged_words = {
         "xxx",
         "ânus",
     ],
+    "ta": english_flagged_words
+    + [
+        "ஓதா",
+        "ஒத்தா",
+        "புண்டை",
+        "ஒம்மாளே",
+        "பக்கி",
+        "கூமுட்டை",
+        "கருமம்",
+        "சனியன்",
+        "கஸ்மாலம்",
+        "சூத்து",
+    ],
     "te": english_flagged_words
     + [
         "గర్భస్రావం",
@@ -1831,437 +1136,69 @@ flagged_words = {
     ],
     "vi": english_flagged_words
     + [
-        "sự phá thai",
-        "hậu môn",
-        "mông",
-        "đồ ngu",
-        "lừa",
-        "lỗ đít",
-        "túi bóng",
-        "những quả bóng",
-        "đồ khốn",
-        "tuyệt vời",
-        "mục sư",
-        "lòng tốt",
-        "chó cái",
-        "dính máu",
-        "công việc thổi",
-        "bollok",
-        "boob",
-        "ngực",
-        "buceta",
-        "ăn mày",
-        "thảm muncher",
-        "sứt mẻ",
-        "cipa",
-        "âm vật",
-        "gà",
-        "gà hút",
-        "gà trống",
-        "coon",
-        "tào lao",
-        "kiêm",
-        "cum",
-        "cunillingus",
+        "cặc",
         "lồn",
-        "chỉ trích",
-        "tinh ranh",
-        "dương vật giả",
-        "dink",
-        "chó-chó",
-        "duche",
-        "đê",
-        "xuất tinh",
-        "fag",
-        "đóng băng",
-        "fagot",
-        "đồ ăn vặt",
-        "người hâm mộ",
-        "nỉ",
-        "thất bại",
-        "mặt bích",
-        "chết tiệt",
-        "quái",
+        "địt",
+        "buồi",
         "đụ",
-        "ôm",
-        "đóng gói fudge",
-        "địa ngục",
-        "có",
-        "sừng",
-        "giật",
-        "kock",
-        "môi âm",
-        "ham muốn",
-        "khổ dâm",
-        "thủ dâm",
-        "mẹ kiếp",
-        "nazi",
-        "người da đen",
-        "người mách nước",
-        "cực khoái",
-        "người mổ",
-        "dương vật",
-        "đi tiểu",
-        "bực mình",
-        "đái",
-        "phân",
-        "khiêu dâm",
-        "nội dung khiêu dâm",
-        "châm",
-        "chích",
-        "pube",
-        "pussies",
-        "âm hộ",
-        "hiếp dâm",
-        "trực tràng",
-        "chậm phát triển",
-        "xé",
-        "người tàn bạo",
-        "vặn vít",
-        "bìu",
-        "tinh dịch",
-        "tình dục",
-        "lông",
-        "xáo trộn",
-        "đồng tính",
-        "cứt",
-        "shite",
-        "ván trượt",
-        "đĩ",
-        "quần lót",
-        "smegma",
-        "xì trum",
-        "con trai",
-        "spac",
-        "spunk",
-        "tinh hoàn",
-        "ăn miếng trả miếng",
-        "titt",
-        "cỏ",
-        "âm đạo",
-        "viagra",
-        "âm môn",
-        "wang",
-        "đã ngủ",
-        "con điếm",
-        "x đánh giá",
-        "xxx",
+        "nứng",
+        "phò",
+        "bú lol",
+        "bú lon",
+        "bú cac",
+        "bú cak",
+        "bú cax",
+        "bú buoi",
+        "bú mút",
+        "húp lol",
+        "húp lon",
+        "húp sò",
+        "nung lol",
+        "nung lon",
+        "nug lon",
+        "nung cac",
+        "nung buoi",
+        "thèm dit",
+        "thèm lon",
+        "thèm lol",
+        "thèm buoi",
+        "thèm dit",
+        "thèm chich",
+        "thèm chịch",
+        "thèm cax",
+        "thèm cak",
+        "móc lol",
+        "móc lon",
+        "buscu",
+        "lol non",
+        "bướm non",
+        "vú to",
+        "vú bự",
+        "lon to",
+        "phim sex",
+        "phim xex",
+        "phim xxx",
     ],
     "zh": english_flagged_words
     + [
-        "13.",
-        "13点",
-        "㞗",
         "三级片",
-        "下三烂",
-        "下贱",
-        "个老子的",
-        "九游",
-        "乳",
         "乳交",
-        "乳头",
-        "乳房",
-        "乳波臀浪",
-        "交配",
-        "仆街",
-        "仆街",
-        "他奶奶",
-        "他奶奶的",
-        "他奶娘的",
-        "他妈",
-        "他妈ㄉ王八蛋",
-        "他妈地",
-        "他妈的",
-        "他娘",
-        "他马的",
-        "你个傻比",
-        "你他马的",
-        "你全家",
-        "你奶奶的",
-        "你她马的",
-        "你妈",
-        "你妈的",
-        "你娘",
-        "你娘卡好",
-        "你娘咧",
-        "你它妈的",
-        "你它马的",
-        "你是鸡",
-        "你是鸭",
-        "你老味",
-        "你老母",
-        "你老闆",
-        "你马的",
-        "做爱",
-        "傻比",
-        "傻逼",
-        "册那",
-        "冚家拎",
-        "冚家鏟",
-        "军妓",
-        "几八",
-        "几叭",
-        "几巴",
-        "几芭",
-        "刚度",
-        "刚瘪三",
-        "包皮",
-        "十三点",
         "卖B",
         "卖比",
         "卖淫",
-        "卵",
-        "卵子",
-        "双峰微颤",
         "口交",
-        "口肯",
-        "叫床",
-        "吃屎",
-        "后庭",
-        "吹箫",
-        "咸家伶",
-        "咸家鏟",
-        "塞你公",
-        "塞你娘",
-        "塞你母",
-        "塞你爸",
-        "塞你老师",
-        "塞你老母",
-        "处女",
-        "外阴",
-        "大卵子",
-        "大卵泡",
-        "大鸡巴",
-        "奶",
-        "奶奶的熊",
-        "奶子",
-        "奸",
-        "奸你",
-        "她妈地",
-        "她妈的",
-        "她马的",
-        "妈B",
-        "妈个B",
-        "妈个比",
-        "妈个老比",
-        "妈妈的",
-        "妈比",
-        "妈的",
-        "妈的B",
-        "妈逼",
-        "妓",
-        "妓女",
-        "妓院",
-        "妳她妈的",
-        "妳妈的",
-        "妳娘的",
-        "妳老母的",
-        "妳马的",
-        "姘头",
         "姣西",
-        "姦",
-        "娘个比",
-        "娘的",
-        "婊子",
-        "婊子养的",
-        "嫖娼",
-        "嫖客",
-        "它妈地",
-        "它妈的",
-        "密洞",
-        "射你",
-        "射精",
-        "小乳头",
-        "小卵子",
-        "小卵泡",
-        "小瘪三",
-        "小肉粒",
-        "小骚比",
-        "小骚货",
-        "小鸡巴",
-        "小鸡鸡",
-        "尻",
-        "屁眼",
-        "屁股",
-        "屄",
-        "屌",
-        "屎忽",
-        "巨乳",
-        "干x娘",
-        "干七八",
-        "干你",
-        "干你妈",
-        "干你娘",
-        "干你老母",
-        "干你良",
-        "干妳妈",
-        "干妳娘",
-        "干妳老母",
-        "干妳马",
-        "干您娘",
-        "干机掰",
-        "干死CS",
-        "干死GM",
-        "干死你",
-        "干死客服",
-        "幹",
-        "强奸",
-        "强奸你",
-        "性",
-        "性交",
-        "性器",
-        "性无能",
-        "性爱",
-        "情色",
-        "想上你",
-        "懆您妈",
-        "懆您娘",
-        "懒8",
-        "懒八",
-        "懒叫",
-        "懒教",
-        "成人",
-        "我操你祖宗十八代",
-        "扒光",
         "打炮",
         "打飞机",
         "抽插",
         "招妓",
         "插你",
         "插死你",
-        "撒尿",
-        "撚",
-        "操你",
-        "操你全家",
-        "操你奶奶",
-        "操你妈",
-        "操你娘",
-        "操你祖宗",
-        "操你老妈",
-        "操你老母",
-        "操妳",
-        "操妳全家",
-        "操妳妈",
-        "操妳娘",
-        "操妳祖宗",
-        "操机掰",
-        "操比",
-        "操逼",
-        "放荡",
-        "日他娘",
-        "日你",
-        "日你妈",
-        "日你老娘",
-        "日你老母",
-        "日批",
-        "月经",
-        "机八",
-        "机巴",
-        "机机歪歪",
-        "杂种",
-        "柒",
-        "浪叫",
-        "淫",
-        "淫乱",
-        "淫妇",
-        "淫棍",
         "淫水",
-        "淫秽",
-        "淫荡",
-        "淫西",
-        "湿透的内裤",
-        "激情",
-        "灨你娘",
-        "烂货",
-        "烂逼",
-        "爛",
-        "狗屁",
-        "狗日",
-        "狗狼养的",
-        "玉杵",
-        "王八蛋",
-        "瓜娃子",
-        "瓜婆娘",
-        "瓜批",
-        "瘪三",
-        "白烂",
-        "白痴",
-        "白癡",
-        "硬膠",
-        "祖宗",
-        "私服",
-        "笨實",
-        "笨蛋",
-        "粉腸",
-        "精子",
-        "老二",
-        "老味",
-        "老母",
-        "老瘪三",
-        "老骚比",
-        "老骚货",
         "肉壁",
         "肉棍子",
-        "肉棒",
         "肉缝",
         "肏",
         "肛交",
-        "肥西",
-        "色情",
-        "花柳",
-        "荡妇",
-        "賤",
-        "贝肉",
-        "贱B",
-        "贱人",
-        "贱货",
-        "贼你妈",
-        "赛你老母",
-        "赛妳阿母",
-        "赣您娘",
-        "躝癱",
-        "轮奸",
-        "迷药",
-        "逼",
-        "逼样",
-        "野鸡",
-        "閪",
-        "阳具",
-        "阳萎",
-        "阴唇",
-        "阴户",
-        "阴核",
-        "阴毛",
-        "阴茎",
-        "阴道",
-        "阴部",
-        "陰莖",
-        "雞巴",
-        "靠北",
-        "靠母",
-        "靠爸",
-        "靠背",
-        "靠腰",
-        "驶你公",
-        "驶你娘",
-        "驶你母",
-        "驶你爸",
-        "驶你老师",
-        "驶你老母",
-        "骚比",
-        "骚货",
-        "骚逼",
-        "鬼公",
-        "鳩",
-        "鸡8",
-        "鸡八",
-        "鸡叭",
-        "鸡吧",
-        "鸡奸",
-        "鸡巴",
-        "鸡芭",
-        "鸡鸡",
-        "龟儿子",
-        "龟头",
     ],
 }

fr.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:301c82d52a8e34f63937afc12970794c8783244c8c0b085a8bbfb0d54dcb9374
+size 2829042764

fr.sp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1b70d5e6556ad245e02ac76919a714ad0b7d288955df65ecd3831a42950b653
+size 942639

fr_examples_with_stats.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dd605b140e7a4c20a00e06c8c70d90333d2559434acd9c182de054d6b53b13b
+size 140859096

id.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e099b6216a558d6c6f6108895e2e13fbc6ffd00b59791d16d6a5f85103ac0be
+size 1847280248

id.sp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b217615a7b185e5e0c967ea5b7156fe149145221e32a54b96dfed15d98b3c807
+size 926624

id_examples_with_stats.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1c05dfc6f847bccf2e79cdb90c0dbb05a7266ae77673cd9f6c3cb811dace8e8
+size 89435039

languages_id.py

@@ -26,8 +26,8 @@ langs_id = [
         "stopwords_id": None,
         "flagged_words_id": None,
         "fasttext_id": "arz",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "arz",
+        "kenlm_id": "arz",
     },
     {
         "lang": "Assamese",
@@ -35,8 +35,8 @@ langs_id = [
         "stopwords_id": None,
         "flagged_words_id": None,
         "fasttext_id": "as",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "as",
+        "kenlm_id": "as",
     },
     {
         "lang": "Bengali",
@@ -80,8 +80,8 @@ langs_id = [
         "stopwords_id": "eu",
         "flagged_words_id": "eu",
         "fasttext_id": "eu",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "eu",
+        "kenlm_id": "eu",
     },
     {
         "lang": "French",
@@ -155,32 +155,23 @@ langs_id = [
         "sentencepiece_id": "pt",
         "kenlm_id": "pt",
     },
-    {
-        "lang": "Somali",
-        "dataset_id": "so",
-        "stopwords_id": "so",
-        "flagged_words_id": None,
-        "fasttext_id": "so",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
-    },
     {
         "lang": "Swahili",
         "dataset_id": "sw",
         "stopwords_id": "sw",
         "flagged_words_id": None,
         "fasttext_id": "sw",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "sw",
+        "kenlm_id": "sw",
     },
     {
         "lang": "Tamil",
         "dataset_id": "ta",
         "stopwords_id": None,
-        "flagged_words_id": None,
+        "flagged_words_id": "ta",
         "fasttext_id": "ta",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "ta",
+        "kenlm_id": "ta",
     },
     {
         "lang": "Telugu",
@@ -188,8 +179,8 @@ langs_id = [
         "stopwords_id": None,
         "flagged_words_id": "te",
         "fasttext_id": "te",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "te",
+        "kenlm_id": "te",
     },
     {
         "lang": "Urdu",
@@ -197,8 +188,8 @@ langs_id = [
         "stopwords_id": "ur",
         "flagged_words_id": None,
         "fasttext_id": "ur",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "ur",
+        "kenlm_id": "ur",
     },
     {
         "lang": "Vietnamese",
@@ -206,8 +197,8 @@ langs_id = [
         "stopwords_id": "vi",
         "flagged_words_id": "vi",
         "fasttext_id": "vi",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "vi",
+        "kenlm_id": "vi",
     },
     {
         "lang": "Yoruba",
@@ -215,8 +206,8 @@ langs_id = [
         "stopwords_id": "yo",
         "flagged_words_id": None,
         "fasttext_id": "yo",
-        "sentencepiece_id": None,
-        "kenlm_id": None,
+        "sentencepiece_id": "yo",
+        "kenlm_id": "yo",
     },
     {
         "lang": "Chinese",

parameters_filtering.py

@@ -28,9 +28,12 @@ parameters_filtering_default = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.4,
@@ -59,9 +62,12 @@ parameters_filtering_af = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.3,
@@ -90,9 +96,12 @@ parameters_filtering_ar = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.45,
@@ -121,9 +130,12 @@ parameters_filtering_arz = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.5,
@@ -152,9 +164,12 @@ parameters_filtering_as = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.25,
@@ -183,9 +198,12 @@ parameters_filtering_bn = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.275,
@@ -214,9 +232,12 @@ parameters_filtering_ca = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.35,
@@ -245,9 +266,12 @@ parameters_filtering_en = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 20,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.4,
@@ -276,9 +300,12 @@ parameters_filtering_es = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.3,
@@ -307,9 +334,12 @@ parameters_filtering_eu = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.3,
@@ -338,9 +368,12 @@ parameters_filtering_fr = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.35,
@@ -369,9 +402,12 @@ parameters_filtering_gu = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.3,
@@ -400,9 +436,12 @@ parameters_filtering_hi = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.35,
@@ -431,9 +470,12 @@ parameters_filtering_id = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.25,
@@ -462,9 +504,12 @@ parameters_filtering_kn = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.25,
@@ -493,9 +538,12 @@ parameters_filtering_ml = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.2,
@@ -524,9 +572,12 @@ parameters_filtering_mr = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.25,
@@ -555,9 +606,12 @@ parameters_filtering_pt = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.3,
@@ -574,37 +628,6 @@ parameters_filtering_pt = {
     "perplexity_max_cutoff": 3000000,
 }
 
-parameters_filtering_so = {
-    "cond_uniform_whitespace": True,
-    "cond_replace_unicode_punctuation": False,
-    "cond_remove_words_with_incorrect_substrings": False,
-    "incorrect_word_substrings": ["http", "www", ".com", "href", "//"],
-    "cond_remove_long_words": False,
-    "length_word_max_cutoff": 1000,
-    "cond_check_number_words": True,
-    "tokenization": False,
-    "strip_characters": special_characters_default,
-    "number_words_min_cutoff": 1,
-    "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
-    "cond_check_special_characters": True,
-    "special_characters": special_characters_default,
-    "special_characters_max_cutoff": 0.3,
-    "cond_words_augmentation": False,
-    "words_augmentation_group_sizes": [],
-    "words_augmentation_join_char": "",
-    "cond_check_stopwords": False,
-    "stopwords_min_cutoff": 0,
-    "cond_check_flagged_words": False,
-    "flagged_words_max_cutoff": 0.2,
-    "cond_check_lang_id": True,
-    "lang_id_min_cutoff": 0.75,
-    "cond_check_perplexity": False,
-    "perplexity_max_cutoff": 3000000,
-}
-
 parameters_filtering_sw = {
     "cond_uniform_whitespace": True,
     "cond_replace_unicode_punctuation": False,
@@ -617,9 +640,12 @@ parameters_filtering_sw = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.275,
@@ -648,9 +674,12 @@ parameters_filtering_ta = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.25,
@@ -679,9 +708,12 @@ parameters_filtering_te = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.25,
@@ -710,9 +742,12 @@ parameters_filtering_ur = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.4,
@@ -741,14 +776,17 @@ parameters_filtering_vi = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.35,
     "cond_words_augmentation": True,
-    "words_augmentation_group_sizes": [2, 3],
+    "words_augmentation_group_sizes": [2],
     "words_augmentation_join_char": " ",
     "cond_check_stopwords": True,
     "stopwords_min_cutoff": 0,
@@ -772,9 +810,12 @@ parameters_filtering_yo = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.3,
@@ -803,14 +844,17 @@ parameters_filtering_zh = {
     "strip_characters": special_characters_default,
     "number_words_min_cutoff": 1,
     "number_words_max_cutoff": 100000,
-    "check_repetitions_removal": True,
-    "repetitions_length": 10,
-    "repetitions_max_cutoff": 0.106,
+    "cond_check_character_repetition_removal": True,
+    "character_repetition_length": 10,
+    "character_repetition_max_cutoff": 0.106,
+    "cond_check_word_repetition_removal": True,
+    "word_repetition_length": 5,
+    "word_repetition_max_cutoff": 0.19,
     "cond_check_special_characters": True,
     "special_characters": special_characters_default,
     "special_characters_max_cutoff": 0.4,
     "cond_words_augmentation": True,
-    "words_augmentation_group_sizes": [2, 3],
+    "words_augmentation_group_sizes": [2],
     "words_augmentation_join_char": "",
     "cond_check_stopwords": False,
     "stopwords_min_cutoff": 0,
@@ -841,7 +885,6 @@ parameters_filtering = {
     "ml": parameters_filtering_ml,
     "mr": parameters_filtering_mr,
     "pt": parameters_filtering_pt,
-    "so": parameters_filtering_so,
     "sw": parameters_filtering_sw,
     "ta": parameters_filtering_ta,
     "te": parameters_filtering_te,

zh_examples_with_stats.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b401f28bfed66f7380c439b311122b734bbf9a1f3012a331a502a83ac18c5ff
-size 74919679
+oid sha256:318cf4641a46c9c7c16fc77171f28475cb8e96935201d3541d493b5231e8d53a
+size 63524762