Upload tokenizer

d2c86d4 verified about 2 months ago

No virus

6.97 kB

	{
	"added_tokens_decoder": {
	"1": {
	"content": "ll",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"3": {
	"content": "oo",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"5": {
	"content": "ee",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"6": {
	"content": "pp",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"9": {
	"content": "tt",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"10": {
	"content": "čč",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"11": {
	"content": "m'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"12": {
	"content": "yy",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"14": {
	"content": "n'n'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"15": {
	"content": "uu",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"16": {
	"content": "ii",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"17": {
	"content": "mm",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"19": {
	"content": "dd",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"22": {
	"content": "kk",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"28": {
	"content": "t'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"29": {
	"content": "d'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"33": {
	"content": "rr",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"34": {
	"content": "hh",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"35": {
	"content": "v'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"37": {
	"content": "nn",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"38": {
	"content": "l'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"39": {
	"content": "s's'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"40": {
	"content": "cc",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"41": {
	"content": "šš",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"42": {
	"content": "f'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"43": {
	"content": "t't'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"46": {
	"content": "n'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"47": {
	"content": "vv",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"49": {
	"content": "l'l'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"50": {
	"content": "s'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"52": {
	"content": "ää",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"53": {
	"content": "aa",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"54": {
	"content": "ss",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"56": {
	"content": "p'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"59": {
	"content": "z'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"62": {
	"content": "r'",
	"lstrip": true,
	"normalized": false,
	"rstrip": true,
	"single_word": false,
	"special": false
	},
	"65": {
	"content": "<s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"66": {
	"content": "</s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"67": {
	"content": "[UNK]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"68": {
	"content": "[PAD]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	}
	},
	"bos_token": "<s>",
	"clean_up_tokenization_spaces": true,
	"do_lower_case": false,
	"eos_token": "</s>",
	"model_max_length": 1000000000000000019884624838656,
	"pad_token": "[PAD]",
	"replace_word_delimiter_char": " ",
	"target_lang": null,
	"tokenizer_class": "Wav2Vec2CTCTokenizer",
	"unk_token": "[UNK]",
	"word_delimiter_token": "\|"
	}