Upload 4 files

data/indus_tokenizer/indus_id_map.json

@@ -0,0 +1,643 @@
+{
+  "T99": 99,
+  "T100": 100,
+  "T101": 101,
+  "T102": 102,
+  "T103": 103,
+  "T104": 104,
+  "T105": 105,
+  "T106": 106,
+  "T107": 107,
+  "T108": 108,
+  "T109": 109,
+  "T110": 110,
+  "T111": 111,
+  "T112": 112,
+  "T113": 113,
+  "T114": 114,
+  "T116": 116,
+  "T117": 117,
+  "T118": 118,
+  "T119": 119,
+  "T120": 120,
+  "T121": 121,
+  "T122": 122,
+  "T123": 123,
+  "T124": 124,
+  "T125": 125,
+  "T126": 126,
+  "T129": 129,
+  "T130": 130,
+  "T133": 133,
+  "T134": 134,
+  "T136": 136,
+  "T137": 137,
+  "T138": 138,
+  "T139": 139,
+  "T140": 140,
+  "T141": 141,
+  "T142": 142,
+  "T143": 143,
+  "T144": 144,
+  "T145": 145,
+  "T146": 146,
+  "T147": 147,
+  "T148": 148,
+  "T149": 149,
+  "T150": 150,
+  "T151": 151,
+  "T152": 152,
+  "T154": 154,
+  "T155": 155,
+  "T156": 156,
+  "T157": 157,
+  "T158": 158,
+  "T159": 159,
+  "T162": 162,
+  "T163": 163,
+  "T164": 164,
+  "T165": 165,
+  "T167": 167,
+  "T168": 168,
+  "T169": 169,
+  "T170": 170,
+  "T171": 171,
+  "T172": 172,
+  "T173": 173,
+  "T174": 174,
+  "T175": 175,
+  "T176": 176,
+  "T177": 177,
+  "T178": 178,
+  "T179": 179,
+  "T180": 180,
+  "T181": 181,
+  "T182": 182,
+  "T183": 183,
+  "T184": 184,
+  "T186": 186,
+  "T187": 187,
+  "T190": 190,
+  "T191": 191,
+  "T192": 192,
+  "T193": 193,
+  "T194": 194,
+  "T195": 195,
+  "T196": 196,
+  "T197": 197,
+  "T198": 198,
+  "T199": 199,
+  "T200": 200,
+  "T201": 201,
+  "T202": 202,
+  "T203": 203,
+  "T204": 204,
+  "T205": 205,
+  "T206": 206,
+  "T208": 208,
+  "T209": 209,
+  "T210": 210,
+  "T211": 211,
+  "T212": 212,
+  "T213": 213,
+  "T214": 214,
+  "T215": 215,
+  "T216": 216,
+  "T217": 217,
+  "T218": 218,
+  "T221": 221,
+  "T222": 222,
+  "T223": 223,
+  "T224": 224,
+  "T225": 225,
+  "T226": 226,
+  "T227": 227,
+  "T228": 228,
+  "T229": 229,
+  "T230": 230,
+  "T231": 231,
+  "T232": 232,
+  "T233": 233,
+  "T234": 234,
+  "T235": 235,
+  "T236": 236,
+  "T238": 238,
+  "T239": 239,
+  "T240": 240,
+  "T241": 241,
+  "T242": 242,
+  "T243": 243,
+  "T244": 244,
+  "T245": 245,
+  "T246": 246,
+  "T247": 247,
+  "T248": 248,
+  "T249": 249,
+  "T250": 250,
+  "T251": 251,
+  "T252": 252,
+  "T253": 253,
+  "T254": 254,
+  "T255": 255,
+  "T256": 256,
+  "T257": 257,
+  "T258": 258,
+  "T259": 259,
+  "T261": 261,
+  "T262": 262,
+  "T263": 263,
+  "T264": 264,
+  "T265": 265,
+  "T267": 267,
+  "T268": 268,
+  "T269": 269,
+  "T270": 270,
+  "T271": 271,
+  "T272": 272,
+  "T273": 273,
+  "T274": 274,
+  "T276": 276,
+  "T277": 277,
+  "T278": 278,
+  "T279": 279,
+  "T280": 280,
+  "T281": 281,
+  "T282": 282,
+  "T283": 283,
+  "T284": 284,
+  "T286": 286,
+  "T287": 287,
+  "T288": 288,
+  "T289": 289,
+  "T290": 290,
+  "T291": 291,
+  "T292": 292,
+  "T293": 293,
+  "T294": 294,
+  "T295": 295,
+  "T296": 296,
+  "T297": 297,
+  "T298": 298,
+  "T299": 299,
+  "T300": 300,
+  "T301": 301,
+  "T302": 302,
+  "T304": 304,
+  "T305": 305,
+  "T306": 306,
+  "T307": 307,
+  "T308": 308,
+  "T310": 310,
+  "T312": 312,
+  "T313": 313,
+  "T314": 314,
+  "T315": 315,
+  "T316": 316,
+  "T317": 317,
+  "T318": 318,
+  "T319": 319,
+  "T320": 320,
+  "T321": 321,
+  "T322": 322,
+  "T323": 323,
+  "T324": 324,
+  "T325": 325,
+  "T326": 326,
+  "T327": 327,
+  "T328": 328,
+  "T329": 329,
+  "T330": 330,
+  "T331": 331,
+  "T332": 332,
+  "T333": 333,
+  "T335": 335,
+  "T336": 336,
+  "T337": 337,
+  "T338": 338,
+  "T339": 339,
+  "T340": 340,
+  "T341": 341,
+  "T342": 342,
+  "T343": 343,
+  "T344": 344,
+  "T345": 345,
+  "T346": 346,
+  "T347": 347,
+  "T348": 348,
+  "T349": 349,
+  "T350": 350,
+  "T351": 351,
+  "T352": 352,
+  "T353": 353,
+  "T354": 354,
+  "T355": 355,
+  "T356": 356,
+  "T357": 357,
+  "T358": 358,
+  "T359": 359,
+  "T360": 360,
+  "T361": 361,
+  "T362": 362,
+  "T363": 363,
+  "T365": 365,
+  "T366": 366,
+  "T367": 367,
+  "T368": 368,
+  "T369": 369,
+  "T370": 370,
+  "T372": 372,
+  "T373": 373,
+  "T374": 374,
+  "T375": 375,
+  "T376": 376,
+  "T377": 377,
+  "T378": 378,
+  "T379": 379,
+  "T380": 380,
+  "T381": 381,
+  "T382": 382,
+  "T383": 383,
+  "T384": 384,
+  "T385": 385,
+  "T386": 386,
+  "T387": 387,
+  "T388": 388,
+  "T389": 389,
+  "T390": 390,
+  "T391": 391,
+  "T392": 392,
+  "T393": 393,
+  "T394": 394,
+  "T395": 395,
+  "T396": 396,
+  "T397": 397,
+  "T399": 399,
+  "T400": 400,
+  "T401": 401,
+  "T402": 402,
+  "T403": 403,
+  "T404": 404,
+  "T405": 405,
+  "T406": 406,
+  "T407": 407,
+  "T408": 408,
+  "T409": 409,
+  "T411": 411,
+  "T412": 412,
+  "T413": 413,
+  "T414": 414,
+  "T415": 415,
+  "T416": 416,
+  "T417": 417,
+  "T418": 418,
+  "T419": 419,
+  "T420": 420,
+  "T421": 421,
+  "T422": 422,
+  "T424": 424,
+  "T425": 425,
+  "T426": 426,
+  "T427": 427,
+  "T428": 428,
+  "T430": 430,
+  "T431": 431,
+  "T432": 432,
+  "T433": 433,
+  "T434": 434,
+  "T435": 435,
+  "T436": 436,
+  "T437": 437,
+  "T438": 438,
+  "T439": 439,
+  "T440": 440,
+  "T441": 441,
+  "T442": 442,
+  "T443": 443,
+  "T444": 444,
+  "T445": 445,
+  "T446": 446,
+  "T448": 448,
+  "T449": 449,
+  "T450": 450,
+  "T451": 451,
+  "T452": 452,
+  "T453": 453,
+  "T455": 455,
+  "T456": 456,
+  "T457": 457,
+  "T458": 458,
+  "T459": 459,
+  "T460": 460,
+  "T461": 461,
+  "T462": 462,
+  "T463": 463,
+  "T464": 464,
+  "T465": 465,
+  "T466": 466,
+  "T467": 467,
+  "T468": 468,
+  "T469": 469,
+  "T470": 470,
+  "T471": 471,
+  "T472": 472,
+  "T473": 473,
+  "T474": 474,
+  "T475": 475,
+  "T476": 476,
+  "T477": 477,
+  "T478": 478,
+  "T479": 479,
+  "T480": 480,
+  "T481": 481,
+  "T482": 482,
+  "T483": 483,
+  "T484": 484,
+  "T485": 485,
+  "T486": 486,
+  "T487": 487,
+  "T489": 489,
+  "T490": 490,
+  "T491": 491,
+  "T492": 492,
+  "T494": 494,
+  "T495": 495,
+  "T496": 496,
+  "T497": 497,
+  "T498": 498,
+  "T499": 499,
+  "T500": 500,
+  "T501": 501,
+  "T502": 502,
+  "T503": 503,
+  "T505": 505,
+  "T506": 506,
+  "T507": 507,
+  "T508": 508,
+  "T509": 509,
+  "T510": 510,
+  "T511": 511,
+  "T512": 512,
+  "T513": 513,
+  "T514": 514,
+  "T515": 515,
+  "T516": 516,
+  "T517": 517,
+  "T518": 518,
+  "T519": 519,
+  "T520": 520,
+  "T521": 521,
+  "T522": 522,
+  "T523": 523,
+  "T525": 525,
+  "T526": 526,
+  "T527": 527,
+  "T528": 528,
+  "T529": 529,
+  "T530": 530,
+  "T531": 531,
+  "T532": 532,
+  "T533": 533,
+  "T534": 534,
+  "T535": 535,
+  "T536": 536,
+  "T537": 537,
+  "T538": 538,
+  "T539": 539,
+  "T541": 541,
+  "T543": 543,
+  "T544": 544,
+  "T545": 545,
+  "T546": 546,
+  "T547": 547,
+  "T548": 548,
+  "T550": 550,
+  "T551": 551,
+  "T552": 552,
+  "T553": 553,
+  "T554": 554,
+  "T555": 555,
+  "T556": 556,
+  "T558": 558,
+  "T559": 559,
+  "T560": 560,
+  "T561": 561,
+  "T563": 563,
+  "T564": 564,
+  "T565": 565,
+  "T566": 566,
+  "T567": 567,
+  "T568": 568,
+  "T569": 569,
+  "T570": 570,
+  "T571": 571,
+  "T572": 572,
+  "T573": 573,
+  "T574": 574,
+  "T575": 575,
+  "T577": 577,
+  "T578": 578,
+  "T579": 579,
+  "T580": 580,
+  "T581": 581,
+  "T582": 582,
+  "T583": 583,
+  "T584": 584,
+  "T585": 585,
+  "T586": 586,
+  "T587": 587,
+  "T588": 588,
+  "T589": 589,
+  "T590": 590,
+  "T591": 591,
+  "T592": 592,
+  "T593": 593,
+  "T594": 594,
+  "T595": 595,
+  "T596": 596,
+  "T597": 597,
+  "T598": 598,
+  "T599": 599,
+  "T600": 600,
+  "T601": 601,
+  "T602": 602,
+  "T604": 604,
+  "T606": 606,
+  "T607": 607,
+  "T608": 608,
+  "T609": 609,
+  "T610": 610,
+  "T611": 611,
+  "T614": 614,
+  "T615": 615,
+  "T616": 616,
+  "T617": 617,
+  "T618": 618,
+  "T620": 620,
+  "T621": 621,
+  "T622": 622,
+  "T624": 624,
+  "T625": 625,
+  "T626": 626,
+  "T627": 627,
+  "T629": 629,
+  "T630": 630,
+  "T631": 631,
+  "T632": 632,
+  "T633": 633,
+  "T635": 635,
+  "T637": 637,
+  "T638": 638,
+  "T639": 639,
+  "T640": 640,
+  "T641": 641,
+  "T642": 642,
+  "T643": 643,
+  "T644": 644,
+  "T645": 645,
+  "T646": 646,
+  "T647": 647,
+  "T648": 648,
+  "T649": 649,
+  "T650": 650,
+  "T652": 652,
+  "T653": 653,
+  "T654": 654,
+  "T655": 655,
+  "T657": 657,
+  "T658": 658,
+  "T660": 660,
+  "T661": 661,
+  "T662": 662,
+  "T663": 663,
+  "T664": 664,
+  "T665": 665,
+  "T666": 666,
+  "T667": 667,
+  "T668": 668,
+  "T672": 672,
+  "T674": 674,
+  "T675": 675,
+  "T676": 676,
+  "T677": 677,
+  "T678": 678,
+  "T679": 679,
+  "T680": 680,
+  "T681": 681,
+  "T682": 682,
+  "T683": 683,
+  "T684": 684,
+  "T685": 685,
+  "T687": 687,
+  "T688": 688,
+  "T689": 689,
+  "T690": 690,
+  "T691": 691,
+  "T692": 692,
+  "T693": 693,
+  "T694": 694,
+  "T695": 695,
+  "T696": 696,
+  "T697": 697,
+  "T698": 698,
+  "T699": 699,
+  "T701": 701,
+  "T702": 702,
+  "T703": 703,
+  "T704": 704,
+  "T705": 705,
+  "T707": 707,
+  "T708": 708,
+  "T709": 709,
+  "T710": 710,
+  "T711": 711,
+  "T712": 712,
+  "T713": 713,
+  "T714": 714,
+  "T715": 715,
+  "T716": 716,
+  "T717": 717,
+  "T719": 719,
+  "T721": 721,
+  "T723": 723,
+  "T725": 725,
+  "T726": 726,
+  "T727": 727,
+  "T728": 728,
+  "T729": 729,
+  "T730": 730,
+  "T731": 731,
+  "T732": 732,
+  "T733": 733,
+  "T734": 734,
+  "T735": 735,
+  "T736": 736,
+  "T737": 737,
+  "T738": 738,
+  "T739": 739,
+  "T741": 741,
+  "T742": 742,
+  "T743": 743,
+  "T744": 744,
+  "T745": 745,
+  "T746": 746,
+  "T747": 747,
+  "T748": 748,
+  "T749": 749,
+  "T750": 750,
+  "T751": 751,
+  "T752": 752,
+  "T753": 753,
+  "T754": 754,
+  "T755": 755,
+  "T756": 756,
+  "T757": 757,
+  "T758": 758,
+  "T759": 759,
+  "T760": 760,
+  "T761": 761,
+  "T762": 762,
+  "T763": 763,
+  "T764": 764,
+  "T765": 765,
+  "T766": 766,
+  "T767": 767,
+  "T768": 768,
+  "T769": 769,
+  "T770": 770,
+  "T771": 771,
+  "T772": 772,
+  "T773": 773,
+  "T774": 774,
+  "T775": 775,
+  "T776": 776,
+  "T777": 777,
+  "T778": 778,
+  "T779": 779,
+  "T780": 780,
+  "T781": 781,
+  "T782": 782,
+  "T783": 783,
+  "T784": 784,
+  "T785": 785,
+  "T786": 786,
+  "T787": 787,
+  "T788": 788,
+  "T789": 789,
+  "T790": 790,
+  "T792": 792,
+  "T793": 793,
+  "T794": 794,
+  "T795": 795,
+  "T797": 797,
+  "T798": 798,
+  "T799": 799,
+  "T801": 801,
+  "T803": 803,
+  "T806": 806,
+  "T807": 807,
+  "T808": 808,
+  "T809": 809,
+  "T810": 810,
+  "T811": 811,
+  "T813": 813
+}

data/indus_tokenizer/tokenizer.json

@@ -0,0 +1,766 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "WhitespaceSplit"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "T99": 5,
+      "T100": 6,
+      "T101": 7,
+      "T102": 8,
+      "T103": 9,
+      "T104": 10,
+      "T105": 11,
+      "T106": 12,
+      "T107": 13,
+      "T108": 14,
+      "T109": 15,
+      "T110": 16,
+      "T111": 17,
+      "T112": 18,
+      "T113": 19,
+      "T114": 20,
+      "T116": 21,
+      "T117": 22,
+      "T118": 23,
+      "T119": 24,
+      "T120": 25,
+      "T121": 26,
+      "T122": 27,
+      "T123": 28,
+      "T124": 29,
+      "T125": 30,
+      "T126": 31,
+      "T129": 32,
+      "T130": 33,
+      "T133": 34,
+      "T134": 35,
+      "T136": 36,
+      "T137": 37,
+      "T138": 38,
+      "T139": 39,
+      "T140": 40,
+      "T141": 41,
+      "T142": 42,
+      "T143": 43,
+      "T144": 44,
+      "T145": 45,
+      "T146": 46,
+      "T147": 47,
+      "T148": 48,
+      "T149": 49,
+      "T150": 50,
+      "T151": 51,
+      "T152": 52,
+      "T154": 53,
+      "T155": 54,
+      "T156": 55,
+      "T157": 56,
+      "T158": 57,
+      "T159": 58,
+      "T162": 59,
+      "T163": 60,
+      "T164": 61,
+      "T165": 62,
+      "T167": 63,
+      "T168": 64,
+      "T169": 65,
+      "T170": 66,
+      "T171": 67,
+      "T172": 68,
+      "T173": 69,
+      "T174": 70,
+      "T175": 71,
+      "T176": 72,
+      "T177": 73,
+      "T178": 74,
+      "T179": 75,
+      "T180": 76,
+      "T181": 77,
+      "T182": 78,
+      "T183": 79,
+      "T184": 80,
+      "T186": 81,
+      "T187": 82,
+      "T190": 83,
+      "T191": 84,
+      "T192": 85,
+      "T193": 86,
+      "T194": 87,
+      "T195": 88,
+      "T196": 89,
+      "T197": 90,
+      "T198": 91,
+      "T199": 92,
+      "T200": 93,
+      "T201": 94,
+      "T202": 95,
+      "T203": 96,
+      "T204": 97,
+      "T205": 98,
+      "T206": 99,
+      "T208": 100,
+      "T209": 101,
+      "T210": 102,
+      "T211": 103,
+      "T212": 104,
+      "T213": 105,
+      "T214": 106,
+      "T215": 107,
+      "T216": 108,
+      "T217": 109,
+      "T218": 110,
+      "T221": 111,
+      "T222": 112,
+      "T223": 113,
+      "T224": 114,
+      "T225": 115,
+      "T226": 116,
+      "T227": 117,
+      "T228": 118,
+      "T229": 119,
+      "T230": 120,
+      "T231": 121,
+      "T232": 122,
+      "T233": 123,
+      "T234": 124,
+      "T235": 125,
+      "T236": 126,
+      "T238": 127,
+      "T239": 128,
+      "T240": 129,
+      "T241": 130,
+      "T242": 131,
+      "T243": 132,
+      "T244": 133,
+      "T245": 134,
+      "T246": 135,
+      "T247": 136,
+      "T248": 137,
+      "T249": 138,
+      "T250": 139,
+      "T251": 140,
+      "T252": 141,
+      "T253": 142,
+      "T254": 143,
+      "T255": 144,
+      "T256": 145,
+      "T257": 146,
+      "T258": 147,
+      "T259": 148,
+      "T261": 149,
+      "T262": 150,
+      "T263": 151,
+      "T264": 152,
+      "T265": 153,
+      "T267": 154,
+      "T268": 155,
+      "T269": 156,
+      "T270": 157,
+      "T271": 158,
+      "T272": 159,
+      "T273": 160,
+      "T274": 161,
+      "T276": 162,
+      "T277": 163,
+      "T278": 164,
+      "T279": 165,
+      "T280": 166,
+      "T281": 167,
+      "T282": 168,
+      "T283": 169,
+      "T284": 170,
+      "T286": 171,
+      "T287": 172,
+      "T288": 173,
+      "T289": 174,
+      "T290": 175,
+      "T291": 176,
+      "T292": 177,
+      "T293": 178,
+      "T294": 179,
+      "T295": 180,
+      "T296": 181,
+      "T297": 182,
+      "T298": 183,
+      "T299": 184,
+      "T300": 185,
+      "T301": 186,
+      "T302": 187,
+      "T304": 188,
+      "T305": 189,
+      "T306": 190,
+      "T307": 191,
+      "T308": 192,
+      "T310": 193,
+      "T312": 194,
+      "T313": 195,
+      "T314": 196,
+      "T315": 197,
+      "T316": 198,
+      "T317": 199,
+      "T318": 200,
+      "T319": 201,
+      "T320": 202,
+      "T321": 203,
+      "T322": 204,
+      "T323": 205,
+      "T324": 206,
+      "T325": 207,
+      "T326": 208,
+      "T327": 209,
+      "T328": 210,
+      "T329": 211,
+      "T330": 212,
+      "T331": 213,
+      "T332": 214,
+      "T333": 215,
+      "T335": 216,
+      "T336": 217,
+      "T337": 218,
+      "T338": 219,
+      "T339": 220,
+      "T340": 221,
+      "T341": 222,
+      "T342": 223,
+      "T343": 224,
+      "T344": 225,
+      "T345": 226,
+      "T346": 227,
+      "T347": 228,
+      "T348": 229,
+      "T349": 230,
+      "T350": 231,
+      "T351": 232,
+      "T352": 233,
+      "T353": 234,
+      "T354": 235,
+      "T355": 236,
+      "T356": 237,
+      "T357": 238,
+      "T358": 239,
+      "T359": 240,
+      "T360": 241,
+      "T361": 242,
+      "T362": 243,
+      "T363": 244,
+      "T365": 245,
+      "T366": 246,
+      "T367": 247,
+      "T368": 248,
+      "T369": 249,
+      "T370": 250,
+      "T372": 251,
+      "T373": 252,
+      "T374": 253,
+      "T375": 254,
+      "T376": 255,
+      "T377": 256,
+      "T378": 257,
+      "T379": 258,
+      "T380": 259,
+      "T381": 260,
+      "T382": 261,
+      "T383": 262,
+      "T384": 263,
+      "T385": 264,
+      "T386": 265,
+      "T387": 266,
+      "T388": 267,
+      "T389": 268,
+      "T390": 269,
+      "T391": 270,
+      "T392": 271,
+      "T393": 272,
+      "T394": 273,
+      "T395": 274,
+      "T396": 275,
+      "T397": 276,
+      "T399": 277,
+      "T400": 278,
+      "T401": 279,
+      "T402": 280,
+      "T403": 281,
+      "T404": 282,
+      "T405": 283,
+      "T406": 284,
+      "T407": 285,
+      "T408": 286,
+      "T409": 287,
+      "T411": 288,
+      "T412": 289,
+      "T413": 290,
+      "T414": 291,
+      "T415": 292,
+      "T416": 293,
+      "T417": 294,
+      "T418": 295,
+      "T419": 296,
+      "T420": 297,
+      "T421": 298,
+      "T422": 299,
+      "T424": 300,
+      "T425": 301,
+      "T426": 302,
+      "T427": 303,
+      "T428": 304,
+      "T430": 305,
+      "T431": 306,
+      "T432": 307,
+      "T433": 308,
+      "T434": 309,
+      "T435": 310,
+      "T436": 311,
+      "T437": 312,
+      "T438": 313,
+      "T439": 314,
+      "T440": 315,
+      "T441": 316,
+      "T442": 317,
+      "T443": 318,
+      "T444": 319,
+      "T445": 320,
+      "T446": 321,
+      "T448": 322,
+      "T449": 323,
+      "T450": 324,
+      "T451": 325,
+      "T452": 326,
+      "T453": 327,
+      "T455": 328,
+      "T456": 329,
+      "T457": 330,
+      "T458": 331,
+      "T459": 332,
+      "T460": 333,
+      "T461": 334,
+      "T462": 335,
+      "T463": 336,
+      "T464": 337,
+      "T465": 338,
+      "T466": 339,
+      "T467": 340,
+      "T468": 341,
+      "T469": 342,
+      "T470": 343,
+      "T471": 344,
+      "T472": 345,
+      "T473": 346,
+      "T474": 347,
+      "T475": 348,
+      "T476": 349,
+      "T477": 350,
+      "T478": 351,
+      "T479": 352,
+      "T480": 353,
+      "T481": 354,
+      "T482": 355,
+      "T483": 356,
+      "T484": 357,
+      "T485": 358,
+      "T486": 359,
+      "T487": 360,
+      "T489": 361,
+      "T490": 362,
+      "T491": 363,
+      "T492": 364,
+      "T494": 365,
+      "T495": 366,
+      "T496": 367,
+      "T497": 368,
+      "T498": 369,
+      "T499": 370,
+      "T500": 371,
+      "T501": 372,
+      "T502": 373,
+      "T503": 374,
+      "T505": 375,
+      "T506": 376,
+      "T507": 377,
+      "T508": 378,
+      "T509": 379,
+      "T510": 380,
+      "T511": 381,
+      "T512": 382,
+      "T513": 383,
+      "T514": 384,
+      "T515": 385,
+      "T516": 386,
+      "T517": 387,
+      "T518": 388,
+      "T519": 389,
+      "T520": 390,
+      "T521": 391,
+      "T522": 392,
+      "T523": 393,
+      "T525": 394,
+      "T526": 395,
+      "T527": 396,
+      "T528": 397,
+      "T529": 398,
+      "T530": 399,
+      "T531": 400,
+      "T532": 401,
+      "T533": 402,
+      "T534": 403,
+      "T535": 404,
+      "T536": 405,
+      "T537": 406,
+      "T538": 407,
+      "T539": 408,
+      "T541": 409,
+      "T543": 410,
+      "T544": 411,
+      "T545": 412,
+      "T546": 413,
+      "T547": 414,
+      "T548": 415,
+      "T550": 416,
+      "T551": 417,
+      "T552": 418,
+      "T553": 419,
+      "T554": 420,
+      "T555": 421,
+      "T556": 422,
+      "T558": 423,
+      "T559": 424,
+      "T560": 425,
+      "T561": 426,
+      "T563": 427,
+      "T564": 428,
+      "T565": 429,
+      "T566": 430,
+      "T567": 431,
+      "T568": 432,
+      "T569": 433,
+      "T570": 434,
+      "T571": 435,
+      "T572": 436,
+      "T573": 437,
+      "T574": 438,
+      "T575": 439,
+      "T577": 440,
+      "T578": 441,
+      "T579": 442,
+      "T580": 443,
+      "T581": 444,
+      "T582": 445,
+      "T583": 446,
+      "T584": 447,
+      "T585": 448,
+      "T586": 449,
+      "T587": 450,
+      "T588": 451,
+      "T589": 452,
+      "T590": 453,
+      "T591": 454,
+      "T592": 455,
+      "T593": 456,
+      "T594": 457,
+      "T595": 458,
+      "T596": 459,
+      "T597": 460,
+      "T598": 461,
+      "T599": 462,
+      "T600": 463,
+      "T601": 464,
+      "T602": 465,
+      "T604": 466,
+      "T606": 467,
+      "T607": 468,
+      "T608": 469,
+      "T609": 470,
+      "T610": 471,
+      "T611": 472,
+      "T614": 473,
+      "T615": 474,
+      "T616": 475,
+      "T617": 476,
+      "T618": 477,
+      "T620": 478,
+      "T621": 479,
+      "T622": 480,
+      "T624": 481,
+      "T625": 482,
+      "T626": 483,
+      "T627": 484,
+      "T629": 485,
+      "T630": 486,
+      "T631": 487,
+      "T632": 488,
+      "T633": 489,
+      "T635": 490,
+      "T637": 491,
+      "T638": 492,
+      "T639": 493,
+      "T640": 494,
+      "T641": 495,
+      "T642": 496,
+      "T643": 497,
+      "T644": 498,
+      "T645": 499,
+      "T646": 500,
+      "T647": 501,
+      "T648": 502,
+      "T649": 503,
+      "T650": 504,
+      "T652": 505,
+      "T653": 506,
+      "T654": 507,
+      "T655": 508,
+      "T657": 509,
+      "T658": 510,
+      "T660": 511,
+      "T661": 512,
+      "T662": 513,
+      "T663": 514,
+      "T664": 515,
+      "T665": 516,
+      "T666": 517,
+      "T667": 518,
+      "T668": 519,
+      "T672": 520,
+      "T674": 521,
+      "T675": 522,
+      "T676": 523,
+      "T677": 524,
+      "T678": 525,
+      "T679": 526,
+      "T680": 527,
+      "T681": 528,
+      "T682": 529,
+      "T683": 530,
+      "T684": 531,
+      "T685": 532,
+      "T687": 533,
+      "T688": 534,
+      "T689": 535,
+      "T690": 536,
+      "T691": 537,
+      "T692": 538,
+      "T693": 539,
+      "T694": 540,
+      "T695": 541,
+      "T696": 542,
+      "T697": 543,
+      "T698": 544,
+      "T699": 545,
+      "T701": 546,
+      "T702": 547,
+      "T703": 548,
+      "T704": 549,
+      "T705": 550,
+      "T707": 551,
+      "T708": 552,
+      "T709": 553,
+      "T710": 554,
+      "T711": 555,
+      "T712": 556,
+      "T713": 557,
+      "T714": 558,
+      "T715": 559,
+      "T716": 560,
+      "T717": 561,
+      "T719": 562,
+      "T721": 563,
+      "T723": 564,
+      "T725": 565,
+      "T726": 566,
+      "T727": 567,
+      "T728": 568,
+      "T729": 569,
+      "T730": 570,
+      "T731": 571,
+      "T732": 572,
+      "T733": 573,
+      "T734": 574,
+      "T735": 575,
+      "T736": 576,
+      "T737": 577,
+      "T738": 578,
+      "T739": 579,
+      "T741": 580,
+      "T742": 581,
+      "T743": 582,
+      "T744": 583,
+      "T745": 584,
+      "T746": 585,
+      "T747": 586,
+      "T748": 587,
+      "T749": 588,
+      "T750": 589,
+      "T751": 590,
+      "T752": 591,
+      "T753": 592,
+      "T754": 593,
+      "T755": 594,
+      "T756": 595,
+      "T757": 596,
+      "T758": 597,
+      "T759": 598,
+      "T760": 599,
+      "T761": 600,
+      "T762": 601,
+      "T763": 602,
+      "T764": 603,
+      "T765": 604,
+      "T766": 605,
+      "T767": 606,
+      "T768": 607,
+      "T769": 608,
+      "T770": 609,
+      "T771": 610,
+      "T772": 611,
+      "T773": 612,
+      "T774": 613,
+      "T775": 614,
+      "T776": 615,
+      "T777": 616,
+      "T778": 617,
+      "T779": 618,
+      "T780": 619,
+      "T781": 620,
+      "T782": 621,
+      "T783": 622,
+      "T784": 623,
+      "T785": 624,
+      "T786": 625,
+      "T787": 626,
+      "T788": 627,
+      "T789": 628,
+      "T790": 629,
+      "T792": 630,
+      "T793": 631,
+      "T794": 632,
+      "T795": 633,
+      "T797": 634,
+      "T798": 635,
+      "T799": 636,
+      "T801": 637,
+      "T803": 638,
+      "T806": 639,
+      "T807": 640,
+      "T808": 641,
+      "T809": 642,
+      "T810": 643,
+      "T811": 644,
+      "T813": 645
+    },
+    "unk_token": "[UNK]"
+  }
+}

data/indus_tokenizer/tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}

data/indus_tokenizer/tokenizer_info.json

@@ -0,0 +1,9 @@
+{
+  "vocab_size": 646,
+  "indus_token_count": 641,
+  "pad_token_id": 0,
+  "cls_token_id": 2,
+  "sep_token_id": 3,
+  "mask_token_id": 4,
+  "unk_token_id": 1
+}