add tokenizer

tokenizer.json

@@ -1 +1 @@
-{"version":"1.0","truncation":{"max_length":512,"strategy":"LongestFirst","stride":0},"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":null,"pre_tokenizer":{"type":"WhitespaceSplit"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"[CLS]":1,"[SEP]":2,"[PAD]":3,"[MASK]":4,"TIME_DELTA=4":5,"TIME_DELTA=2":6,"BAR_START":7,"BAR_END":8,"TRACK_START":9,"TRACK_END":10,"NOTE_OFF=67":11,"NOTE_ON=67":12,"NOTE_ON=65":13,"NOTE_OFF=65":14,"NOTE_ON=62":15,"NOTE_OFF=62":16,"NOTE_OFF=60":17,"NOTE_ON=60":18,"TIME_DELTA=8":19,"NOTE_OFF=64":20,"NOTE_ON=64":21,"NOTE_OFF=63":22,"NOTE_ON=63":23,"NOTE_OFF=66":24,"NOTE_ON=66":25,"NOTE_ON=69":26,"NOTE_OFF=69":27,"NOTE_OFF=68":28,"NOTE_ON=68":29,"NOTE_OFF=61":30,"NOTE_ON=61":31,"NOTE_ON=58":32,"NOTE_OFF=58":33,"NOTE_ON=55":34,"NOTE_OFF=55":35,"NOTE_ON=70":36,"NOTE_OFF=70":37,"NOTE_ON=59":38,"NOTE_OFF=59":39,"NOTE_ON=57":40,"NOTE_OFF=57":41,"NOTE_ON=56":42,"NOTE_OFF=56":43,"NOTE_OFF=53":44,"NOTE_ON=53":45,"NOTE_OFF=71":46,"NOTE_ON=71":47,"NOTE_ON=72":48,"NOTE_OFF=72":49,"NOTE_OFF=54":50,"NOTE_ON=54":51,"NOTE_ON=52":52,"NOTE_OFF=52":53,"NOTE_OFF=50":54,"NOTE_ON=50":55,"NOTE_ON=51":56,"NOTE_OFF=51":57,"NOTE_ON=48":58,"NOTE_OFF=48":59,"NOTE_ON=73":60,"NOTE_OFF=73":61,"NOTE_OFF=74":62,"NOTE_ON=74":63,"NOTE_ON=49":64,"NOTE_OFF=49":65,"NOTE_ON=47":66,"NOTE_OFF=47":67,"NOTE_ON=75":68,"NOTE_OFF=75":69,"NOTE_OFF=46":70,"NOTE_ON=46":71,"INST=48":72,"INST=24":73,"INST=0":74,"STYLE=JSFAKES":75,"INST=32":76,"GENRE=JSFAKES":77,"PIECE_START":78,"PIECE_END":79,"TIME_DELTA=1":80,"NOTE_OFF=45":81,"NOTE_ON=45":82,"NOTE_ON=76":83,"NOTE_OFF=76":84,"TIME_DELTA=6":85,"NOTE_OFF=44":86,"NOTE_ON=44":87,"TIME_DELTA=12":88,"NOTE_ON=43":89,"NOTE_OFF=43":90,"NOTE_OFF=77":91,"NOTE_ON=77":92,"NOTE_OFF=78":93,"NOTE_ON=78":94,"NOTE_OFF=41":95,"NOTE_ON=41":96,"NOTE_ON=42":97,"NOTE_OFF=42":98,"NOTE_ON=79":99,"NOTE_OFF=79":100,"NOTE_ON=40":101,"NOTE_OFF=40":102,"TIME_DELTA=16":103,"NOTE_ON=39":104,"NOTE_OFF=39":105,"NOTE_OFF=80":106,"NOTE_ON=80":107,"TIME_DELTA=3":108,"NOTE_ON=38":109,"NOTE_OFF=38":110,"NOTE_ON=37":111,"NOTE_OFF=37":112,"NOTE_ON=36":113,"NOTE_OFF=36":114,"NOTE_OFF=81":115,"NOTE_ON=81":116,"TIME_DELTA=7":117,"TIME_DELTA=10":118},"unk_token":"[UNK]"}}
+{"version":"1.0","truncation":{"max_length":512,"strategy":"LongestFirst","stride":0},"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":null,"pre_tokenizer":{"type":"WhitespaceSplit"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"[CLS]":1,"[SEP]":2,"[PAD]":3,"[MASK]":4,"TIME_DELTA=4":5,"TIME_DELTA=2":6,"BAR_END":7,"BAR_START":8,"TRACK_START":9,"TRACK_END":10,"NOTE_OFF=67":11,"NOTE_ON=67":12,"NOTE_ON=65":13,"NOTE_OFF=65":14,"NOTE_ON=62":15,"NOTE_OFF=62":16,"NOTE_ON=60":17,"NOTE_OFF=60":18,"TIME_DELTA=8":19,"NOTE_OFF=64":20,"NOTE_ON=64":21,"NOTE_ON=63":22,"NOTE_OFF=63":23,"NOTE_OFF=66":24,"NOTE_ON=66":25,"NOTE_OFF=69":26,"NOTE_ON=69":27,"NOTE_OFF=68":28,"NOTE_ON=68":29,"NOTE_ON=61":30,"NOTE_OFF=61":31,"NOTE_OFF=58":32,"NOTE_ON=58":33,"NOTE_ON=55":34,"NOTE_OFF=55":35,"NOTE_OFF=70":36,"NOTE_ON=70":37,"NOTE_OFF=59":38,"NOTE_ON=59":39,"NOTE_OFF=57":40,"NOTE_ON=57":41,"NOTE_ON=56":42,"NOTE_OFF=56":43,"NOTE_OFF=53":44,"NOTE_ON=53":45,"NOTE_OFF=71":46,"NOTE_ON=71":47,"NOTE_ON=72":48,"NOTE_OFF=72":49,"NOTE_ON=54":50,"NOTE_OFF=54":51,"NOTE_ON=52":52,"NOTE_OFF=52":53,"NOTE_OFF=50":54,"NOTE_ON=50":55,"NOTE_ON=51":56,"NOTE_OFF=51":57,"NOTE_ON=48":58,"NOTE_OFF=48":59,"NOTE_OFF=73":60,"NOTE_ON=73":61,"NOTE_OFF=74":62,"NOTE_ON=74":63,"NOTE_OFF=49":64,"NOTE_ON=49":65,"NOTE_ON=47":66,"NOTE_OFF=47":67,"NOTE_ON=75":68,"NOTE_OFF=75":69,"NOTE_OFF=46":70,"NOTE_ON=46":71,"INST=24":72,"PIECE_END":73,"INST=48":74,"GENRE=JSFAKES":75,"INST=0":76,"PIECE_START":77,"STYLE=JSFAKES":78,"INST=32":79,"TIME_DELTA=1":80,"NOTE_OFF=45":81,"NOTE_ON=45":82,"NOTE_ON=76":83,"NOTE_OFF=76":84,"TIME_DELTA=6":85,"NOTE_OFF=44":86,"NOTE_ON=44":87,"TIME_DELTA=12":88,"NOTE_ON=43":89,"NOTE_OFF=43":90,"NOTE_OFF=77":91,"NOTE_ON=77":92,"NOTE_OFF=78":93,"NOTE_ON=78":94,"NOTE_OFF=41":95,"NOTE_ON=41":96,"NOTE_ON=42":97,"NOTE_OFF=42":98,"NOTE_ON=79":99,"NOTE_OFF=79":100,"NOTE_OFF=40":101,"NOTE_ON=40":102,"TIME_DELTA=16":103,"NOTE_ON=39":104,"NOTE_OFF=39":105,"NOTE_OFF=80":106,"NOTE_ON=80":107,"TIME_DELTA=3":108,"NOTE_OFF=38":109,"NOTE_ON=38":110,"NOTE_OFF=37":111,"NOTE_ON=37":112,"NOTE_ON=36":113,"NOTE_OFF=36":114,"NOTE_OFF=81":115,"NOTE_ON=81":116,"TIME_DELTA=7":117,"TIME_DELTA=10":118},"unk_token":"[UNK]"}}