ManukyanD
/

flan-t5-for-text2sql

Text2Text Generation

Inference Endpoints

text-generation-inference

Model card Files Files and versions Community

ManukyanD commited on Apr 20

Commit

0aae421

•

1 Parent(s): f6cc33c

Upload tokenizer

Files changed (2) hide show

tokenizer.json +4 -4
tokenizer_config.json +7 -0

tokenizer.json CHANGED Viewed

@@ -989,8 +989,8 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
   },
   "post_processor": {
     "type": "TemplateProcessing",
@@ -1049,8 +1049,8 @@
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
   },
   "model": {
     "type": "Unigram",

   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
   },
   "post_processor": {
     "type": "TemplateProcessing",
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
   },
   "model": {
     "type": "Unigram",

tokenizer_config.json CHANGED Viewed

@@ -954,9 +954,16 @@
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},
   "tokenizer_class": "T5Tokenizer",
   "unk_token": "<unk>"
 }

   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,
+  "max_length": 512,
   "model_max_length": 512,
+  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sp_model_kwargs": {},
+  "stride": 0,
   "tokenizer_class": "T5Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }