Upload tokenizer

README.md

@@ -1,11 +1,11 @@
 ---
+language:
+- tr
 library_name: transformers
 base_model: boun-tabi-LMG/TURNA
 datasets:
 - GGLab/GECTurk
 - mcemilg/GECTurk-generation
-language:
-- tr
 ---
 
 # Model Card for Model ID

tokenizer.json

@@ -964,15 +964,15 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
+    "prepend_scheme": "always",
+    "split": true
   },
   "post_processor": null,
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
+    "prepend_scheme": "always",
+    "split": true
   },
   "model": {
     "type": "Unigram",

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<PAD>",
@@ -939,10 +941,13 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<EOS>",
   "extra_ids": 100,
+  "max_length": 45,
   "model_max_length": 1024,
   "pad_token": "<PAD>",
-  "padding_side": "right",
+  "padding_side": "left",
+  "stride": 0,
   "tokenizer_class": "PreTrainedTokenizerFast",
   "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<UNK>"
 }