Uploaded new saved tokenizer

added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "<|endoftext|>": 255,
+  "[SPACE]": 2,
+  "[STOP]": 0,
+  "[UNK]": 1
+}

merges.txt

@@ -213,4 +213,3 @@ u se
 b l
 sa id
 o ver
-ge t

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "[STOP]",
+  "pad_token": "[STOP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,49 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": true,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[STOP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "2": {
+      "content": "[SPACE]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "[STOP]",
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[STOP]",
+  "tokenizer_class": "ClvpTokenizer",
+  "tokenizer_file": "/home/susnato/.cache/huggingface/hub/models--susnato--clvp_dev/snapshots/75fdab928ec8ef23e0f66bb0bf70518989388456/tokenizer.json",
+  "unk_token": "[UNK]"
+}

vocab.json

@@ -1 +1,257 @@
-{"te": 136, "con": 135, "wh": 72, "ong": 222, "pp": 246, "ough": 202, "from": 154, "there": 172, "in": 41, "ul": 167, "ca": 183, "said": 252, "who": 214, "ou": 45, "id": 83, "ter": 114, "were": 184, "m": 26, "up": 152, "ant": 159, "en": 50, "ind": 168, "ere": 138, "into": 228, "el": 249, "ow": 69, "whi": 230, "y": 38, "th": 40, "n": 27, "od": 207, "ity": 240, "ate": 186, "you": 74, "ag": 143, "as": 55, "for": 87, "ha": 102, "so": 123, "f": 19, "if": 140, "ty": 210, "em": 225, "u": 34, "der": 234, "at": 48, "ous": 206, "but": 128, "have": 148, "li": 75, "co": 174, "back": 241, "r": 31, "way": 180, "e": 18, "j": 23, "ur": 121, "mo": 115, "this": 147, "ab": 109, "k": 24, "me": 80, "ill": 216, "o": 28, "was": 81, "some": 171, "pe": 124, "lo": 99, "res": 175, "we": 100, "bl": 251, "p": 29, "what": 193, "[SPACE]": 2, "ad": 68, "t": 33, "ld": 79, "use": 250, "z": 39, "could": 221, "al": 56, "ve": 76, "ay": 88, "ke": 93, "ts": 192, "ac": 77, "ain": 137, "pl": 178, "ch": 71, "qu": 194, "ight": 196, "w": 36, "ation": 158, "ir": 98, "ine": 203, "ri": 101, "il": 151, "h": 21, "my": 125, "just": 226, "ound": 189, "can": 201, "when": 199, "red": 219, "'": 4, "tion": 181, "do": 134, "any": 229, "had": 116, "go": 119, "oun": 146, "-": 8, "their": 224, "b": 15, "es": 61, "ther": 131, "st": 63, "sh": 120, "ap": 155, "ok": 166, "i": 22, "per": 205, "of": 58, "er": 44, "now": 145, "ak": 213, "ting": 242, "with": 103, "they": 141, "been": 237, "ted": 188, "ll": 84, "ut": 96, "am": 122, "ome": 129, "more": 244, "su": 133, "a": 14, "ack": 157, "them": 217, "fo": 182, "know": 209, "by": 185, "po": 139, "ce": 117, ";": 12, "ally": 195, "then": 247, "s": 32, "no": 95, "se": 66, "about": 215, "im": 106, "ti": 78, "our": 160, "ye": 220, "br": 243, "the": 42, "x": 37, "ard": 177, "gr": 198, "ink": 239, "ge": 112, "own": 190, "ide": 208, "gh": 82, "fe": 132, "ven": 200, "!": 3, "did": 235, "(": 5, "very": 211, "wi": 85, "his": 94, ")": 6, "ne": 111, "sa": 150, "tw": 232, "ly": 70, "fr": 130, "man": 176, "get": 254, "that": 73, "om": 65, "si": 212, "ast": 233, "l": 25, "us": 142, "an": 43, "and": 53, "end": 204, "g": 20, "to": 51, "one": 110, "ould": 113, "ver": 90, "ro": 89, "op": 161, "sp": 248, "on": 47, "him": 156, "c": 16, "ar": 59, "out": 105, "/": 10, "or": 57, "le": 64, "ion": 107, "com": 170, "would": 191, "ed": 49, "all": 108, "ent": 86, "thing": 153, "q": 30, "it": 60, "[UNK]": 1, "her": 92, ",": 7, "ex": 169, "ake": 245, "ers": 173, "d": 17, "ing": 52, "not": 149, "[STOP]": 0, ".": 9, "like": 162, "she": 118, "be": 67, "ace": 238, "un": 97, "tr": 144, "are": 127, "v": 35, "ic": 91, "is": 54, "over": 253, "your": 223, "ght": 104, "?": 13, "ie": 236, "he": 62, "est": 218, "ust": 163, "re": 46, "wor": 179, ":": 11, "de": 126, "other": 227, "ess": 164, "pro": 187, "um": 231, "ck": 197, "bo": 165}
+{
+  "!": 3,
+  "'": 4,
+  "(": 5,
+  ")": 6,
+  ",": 7,
+  "-": 8,
+  ".": 9,
+  "/": 10,
+  ":": 11,
+  ";": 12,
+  "?": 13,
+  "[SPACE]": 2,
+  "[STOP]": 0,
+  "[UNK]": 1,
+  "a": 14,
+  "ab": 109,
+  "about": 215,
+  "ac": 77,
+  "ace": 238,
+  "ack": 157,
+  "ad": 68,
+  "ag": 143,
+  "ain": 137,
+  "ak": 213,
+  "ake": 245,
+  "al": 56,
+  "all": 108,
+  "ally": 195,
+  "am": 122,
+  "an": 43,
+  "and": 53,
+  "ant": 159,
+  "any": 229,
+  "ap": 155,
+  "ar": 59,
+  "ard": 177,
+  "are": 127,
+  "as": 55,
+  "ast": 233,
+  "at": 48,
+  "ate": 186,
+  "ation": 158,
+  "ay": 88,
+  "b": 15,
+  "back": 241,
+  "be": 67,
+  "been": 237,
+  "bl": 251,
+  "bo": 165,
+  "br": 243,
+  "but": 128,
+  "by": 185,
+  "c": 16,
+  "ca": 183,
+  "can": 201,
+  "ce": 117,
+  "ch": 71,
+  "ck": 197,
+  "co": 174,
+  "com": 170,
+  "con": 135,
+  "could": 221,
+  "d": 17,
+  "de": 126,
+  "der": 234,
+  "did": 235,
+  "do": 134,
+  "e": 18,
+  "ed": 49,
+  "el": 249,
+  "em": 225,
+  "en": 50,
+  "end": 204,
+  "ent": 86,
+  "er": 44,
+  "ere": 138,
+  "ers": 173,
+  "es": 61,
+  "ess": 164,
+  "est": 218,
+  "ex": 169,
+  "f": 19,
+  "fe": 132,
+  "fo": 182,
+  "for": 87,
+  "fr": 130,
+  "from": 154,
+  "g": 20,
+  "ge": 112,
+  "get": 254,
+  "gh": 82,
+  "ght": 104,
+  "go": 119,
+  "gr": 198,
+  "h": 21,
+  "ha": 102,
+  "had": 116,
+  "have": 148,
+  "he": 62,
+  "her": 92,
+  "him": 156,
+  "his": 94,
+  "i": 22,
+  "ic": 91,
+  "id": 83,
+  "ide": 208,
+  "ie": 236,
+  "if": 140,
+  "ight": 196,
+  "il": 151,
+  "ill": 216,
+  "im": 106,
+  "in": 41,
+  "ind": 168,
+  "ine": 203,
+  "ing": 52,
+  "ink": 239,
+  "into": 228,
+  "ion": 107,
+  "ir": 98,
+  "is": 54,
+  "it": 60,
+  "ity": 240,
+  "j": 23,
+  "just": 226,
+  "k": 24,
+  "ke": 93,
+  "know": 209,
+  "l": 25,
+  "ld": 79,
+  "le": 64,
+  "li": 75,
+  "like": 162,
+  "ll": 84,
+  "lo": 99,
+  "ly": 70,
+  "m": 26,
+  "man": 176,
+  "me": 80,
+  "mo": 115,
+  "more": 244,
+  "my": 125,
+  "n": 27,
+  "ne": 111,
+  "no": 95,
+  "not": 149,
+  "now": 145,
+  "o": 28,
+  "od": 207,
+  "of": 58,
+  "ok": 166,
+  "om": 65,
+  "ome": 129,
+  "on": 47,
+  "one": 110,
+  "ong": 222,
+  "op": 161,
+  "or": 57,
+  "other": 227,
+  "ou": 45,
+  "ough": 202,
+  "ould": 113,
+  "oun": 146,
+  "ound": 189,
+  "our": 160,
+  "ous": 206,
+  "out": 105,
+  "over": 253,
+  "ow": 69,
+  "own": 190,
+  "p": 29,
+  "pe": 124,
+  "per": 205,
+  "pl": 178,
+  "po": 139,
+  "pp": 246,
+  "pro": 187,
+  "q": 30,
+  "qu": 194,
+  "r": 31,
+  "re": 46,
+  "red": 219,
+  "res": 175,
+  "ri": 101,
+  "ro": 89,
+  "s": 32,
+  "sa": 150,
+  "said": 252,
+  "se": 66,
+  "sh": 120,
+  "she": 118,
+  "si": 212,
+  "so": 123,
+  "some": 171,
+  "sp": 248,
+  "st": 63,
+  "su": 133,
+  "t": 33,
+  "te": 136,
+  "ted": 188,
+  "ter": 114,
+  "th": 40,
+  "that": 73,
+  "the": 42,
+  "their": 224,
+  "them": 217,
+  "then": 247,
+  "ther": 131,
+  "there": 172,
+  "they": 141,
+  "thing": 153,
+  "this": 147,
+  "ti": 78,
+  "ting": 242,
+  "tion": 181,
+  "to": 51,
+  "tr": 144,
+  "ts": 192,
+  "tw": 232,
+  "ty": 210,
+  "u": 34,
+  "ul": 167,
+  "um": 231,
+  "un": 97,
+  "up": 152,
+  "ur": 121,
+  "us": 142,
+  "use": 250,
+  "ust": 163,
+  "ut": 96,
+  "v": 35,
+  "ve": 76,
+  "ven": 200,
+  "ver": 90,
+  "very": 211,
+  "w": 36,
+  "was": 81,
+  "way": 180,
+  "we": 100,
+  "were": 184,
+  "wh": 72,
+  "what": 193,
+  "when": 199,
+  "whi": 230,
+  "who": 214,
+  "wi": 85,
+  "with": 103,
+  "wor": 179,
+  "would": 191,
+  "x": 37,
+  "y": 38,
+  "ye": 220,
+  "you": 74,
+  "your": 223,
+  "z": 39
+}