import json
from transformers import AutoTokenizer, BloomTokenizerFast


# tokenizer = AutoTokenizer.from_pretrained("tokenizer", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained("moss-moon-003-sft", trust_remote_code=True)

print("vocab size:", tokenizer.vocab_size)


tokens = [  1639,    389,    281,   9552,   8796,   3025,   1438,    318,    337,
         18420,     13,    198,     12,    337,  18420,    318,    257,   3453,
           864,   3303,   2746,    326,    318,   4166,    416,    376,    463,
           272,   2059,     13,    632,    318,   3562,    284,    307,   7613,
            11,   5508,     11,    290,  23585,     13,    198,     12,    337,
         18420,    460,   1833,    290,  10996,   6562,   1473,    287,    262,
          3303,   7147,    416,    262,   2836,    884,    355,   3594,    290,
           220,  54119,     13,    337,  18420,    460,   1620,    597,   3303,
            12,   3106,   8861,     13,    198,     12,    337,  18420,   1276,
         11148,    284,   2112,   1997,   3519,    284,    663,  36454,     11,
          7729,     11,    393,   3173,     13,    198,     12,   6363,   9109,
          1276,    407,    307,  13443,     11,  10458,   2870,     11,  22066,
            11,   8381,     11,    572,     12,  26652,     11,    393,   6110,
            13,    198,     12,    632,    815,   3368,   3501,  19088,   9317,
           475,   8814,    319,   9432,   6419,    393,  20144,    588,    366,
           259,    428,   4732,    257,   1692,   1244,    910,   9313,     11,
           366,  11246,    661,   1244,    892,   9313,     11,   3503,     13,
           198,     12,   6363,   9109,   1276,    635,    307,   3967,     11,
         23507,     11,   3499,     11,  17774,     11,    290,  11932,     13,
           198,     12,    632,    460,   2148,   3224,   5981,   3307,    284,
          3280,    287,     12,  18053,    290,   8569,   2280,   9505,   4517,
          2480,   7612,     13,    198,     12,    632,   8453,   4340,    290,
         18178,    262,   2836,    338,  13052,    611,    262,   2836,   3376,
            82,    262,  11491,   3280,   7560,    416,    337,  18420,     13,
           198,  15610,   5738,    290,   4899,    326,    337,  18420,    460,
          8588,     13,    198,     27,     91,  20490,     91,  31175,  59163,
         50331,    220, 106067,    220,    198,     27,     91,     44,  18420,
            91,  31175,  10545,    224,    101,  50331,  50422,  52746,     44,
         18420,  50257,  52858,  50264,  58623,  55367,  51131,  50379,    220,
        106068,    198,     27,     91,  20490,     91,  31175,  10545,    236,
           101,  52047,  49390,  50428,  65292,  51916, 106067,    198,     27,
            91,     44,  18420,     91,  31175,  10263,    121,    241,  50368,
         50427,  50422,  62342,  49390,  50428,  51137,  66559,  65292,  51916,
         50313,    198,    198,     16,  64748,  14585,  60579,  80526,  54384,
         14585,     25,    317,   4687,  28032,  56866,  50614,  56456,  50573,
          9129,  51713,  50809,  67542,  63661,  50257,  69292,  52794,  50261,
         54740,  55061,  56164,  50257,  51206,  52427,  70255,  54261,  63632,
         50257,  50515,  56999,  72855,  52617,  55274,  16764,    198,    198,
            17,  64748,  51236,  53092,  61367,  54384,  47520,  21529,  56866,
         50614,  51700,  88026,   9129,  96919,  63661,  50257,  56723,  52427,
         52179,  77566,  50257,  52794,  50387,  52731,  86875,  53312,  52064,
         16764,    198,    198,     18,  64748,  62847,  56604,  54384,   8248,
          6176,  50394,  52189,  50313,  50614,  61283,   9129,  53459,  66122,
         63661,  50257,  56723,  52427,  79535,  72227,  40792,  50257,  51436,
         67464,  21410,  55794,  53312,  53340,  16764,    198,    198,     19,
         64748,  73713,  55794,  54384,    464,  24936,  56866,  50614,  50865,
         53701,  50285,  78675,   9129,  53850,  53534,  60431,  63661,  50257,
         56723,  52427,  55903,  51113,  97202,  51113,  53312,  57832,  16764,
           198,    198,     20,  64748,  92567,  54384,  44501,  56866,  50614,
         50363,  88026,   9129,  96919,  63661,  50257,  56723,  50890,  50810,
         96601,  56254,  50584,  56035,  57043,  58967,  66120,  54999,  50956,
         52707,  55409,  16764, 106068]
decode_line = tokenizer.decode(tokens)
print(decode_line)


for token in tokens:
    print(token, tokenizer.decode([token]))