Push model using huggingface_hub.

tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json

@@ -77,7 +77,7 @@
     },
     {
       "id": 8,
-      "content": "<MOLECULAR_ENTITY_ANTIGEN>",
+      "content": "<INTERNAL_0>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2318,7 +2318,7 @@
     },
     {
       "id": 257,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIGEN>",
+      "content": "<INTERNAL_17>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2327,7 +2327,7 @@
     },
     {
       "id": 258,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_15>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2336,7 +2336,7 @@
     },
     {
       "id": 259,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_16>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2417,7 +2417,7 @@
     },
     {
       "id": 268,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>",
+      "content": "<INTERNAL_7>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2426,7 +2426,7 @@
     },
     {
       "id": 269,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>",
+      "content": "<INTERNAL_6>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2435,7 +2435,7 @@
     },
     {
       "id": 270,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>",
+      "content": "<INTERNAL_9>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2444,7 +2444,7 @@
     },
     {
       "id": 271,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>",
+      "content": "<INTERNAL_5>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2453,7 +2453,7 @@
     },
     {
       "id": 272,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>",
+      "content": "<INTERNAL_8>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2462,7 +2462,7 @@
     },
     {
       "id": 273,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>",
+      "content": "<INTERNAL_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2570,7 +2570,7 @@
     },
     {
       "id": 285,
-      "content": "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>",
+      "content": "<INTERNAL_14>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2669,7 +2669,7 @@
     },
     {
       "id": 296,
-      "content": "<CDR3_REGION>",
+      "content": "<INTERNAL_13>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2678,7 +2678,7 @@
     },
     {
       "id": 297,
-      "content": "<GENERAL_CHAIN>",
+      "content": "<INTERNAL_12>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2828,33 +2828,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 314,
-      "content": "<BBBP>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 315,
-      "content": "<FDA_APPR>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 316,
-      "content": "<HIV_ACTIVITY>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -2879,10 +2852,10 @@
       "<EOS>": 5,
       "<MOLECULAR_ENTITY>": 6,
       "<GLOBAL_INTERACTION_ATTRIBUTES>": 7,
-      "<MOLECULAR_ENTITY_ANTIGEN>": 8,
+      "<INTERNAL_0>": 8,
       "<MOLECULAR_ENTITY_EPITOPE>": 9,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>": 10,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>": 11,
+      "<INTERNAL_2>": 10,
+      "<INTERNAL_3>": 11,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CHAIN>": 12,
       "<MOLECULAR_ENTITY_TCR_BETA_VDJ>": 13,
       "<MOLECULAR_ENTITY_TCR_BETA_CDR3>": 14,
@@ -3128,9 +3101,9 @@
       "<SENTINEL_ID_197>": 254,
       "<SENTINEL_ID_198>": 255,
       "<SENTINEL_ID_199>": 256,
-      "<MOLECULAR_ENTITY_TYPE_ANTIGEN>": 257,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>": 258,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>": 259,
+      "<INTERNAL_17>": 257,
+      "<INTERNAL_15>": 258,
+      "<INTERNAL_16>": 259,
       "<ATTRIBUTE_ORGANISM>": 260,
       "<ATTRIBUTE_ORGANISM_HUMAN>": 261,
       "<ATTRIBUTE_ORGANISM_RABBIT>": 262,
@@ -3139,12 +3112,12 @@
       "<ATTRIBUTE_ORGANISM_MONKEY>": 265,
       "<ATTRIBUTE_ORGANISM_CAMEL>": 266,
       "<EPITOPE_PARATOPE_PREDICTION>": 267,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>": 268,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>": 269,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>": 270,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>": 271,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>": 272,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>": 273,
+      "<INTERNAL_7>": 268,
+      "<INTERNAL_6>": 269,
+      "<INTERNAL_9>": 270,
+      "<INTERNAL_5>": 271,
+      "<INTERNAL_8>": 272,
+      "<INTERNAL_4>": 273,
       "<MOLECULAR_ENTITY_GENERAL_PROTEIN>": 274,
       "<TIMESTEP>": 275,
       "<DIFFUSION>": 276,
@@ -3156,7 +3129,7 @@
       "<BACKSPACE>": 282,
       "<SEQUENCE_NATURAL_START>": 283,
       "<NOOP>": 284,
-      "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>": 285,
+      "<INTERNAL_14>": 285,
       "<MOLECULAR_ENTITY_SMALL_MOLECULE>": 286,
       "<MOLECULAR_ENTITY_CELL_GENE_EXPRESSION_RANKED>": 287,
       "<CELL_TYPE_CLASS>": 288,
@@ -3167,8 +3140,8 @@
       "<MOLECULAR_ENTITY_PROTEIN_CHAIN>": 293,
       "<COMPLEX_ENTITY>": 294,
       "<ALTERNATIVE>": 295,
-      "<CDR3_REGION>": 296,
-      "<GENERAL_CHAIN>": 297,
+      "<INTERNAL_13>": 296,
+      "<INTERNAL_12>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
@@ -3185,9 +3158,6 @@
       "<AUTOENCODER_LATENT_SAMPLED_Z>": 311,
       "<AUTOENCODER_TASK>": 312,
       "<DECODED_FROM_LATENT>": 313,
-      "<BBBP>": 314,
-      "<FDA_APPR>": 315,
-      "<HIV_ACTIVITY>": 316,
       "#": 527,
       "%": 528,
       "(": 529,

tokenizer/cell_attributes_tokenizer.json

@@ -77,7 +77,7 @@
     },
     {
       "id": 8,
-      "content": "<MOLECULAR_ENTITY_ANTIGEN>",
+      "content": "<INTERNAL_0>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2318,7 +2318,7 @@
     },
     {
       "id": 257,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIGEN>",
+      "content": "<INTERNAL_17>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2327,7 +2327,7 @@
     },
     {
       "id": 258,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_15>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2336,7 +2336,7 @@
     },
     {
       "id": 259,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_16>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2417,7 +2417,7 @@
     },
     {
       "id": 268,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>",
+      "content": "<INTERNAL_7>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2426,7 +2426,7 @@
     },
     {
       "id": 269,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>",
+      "content": "<INTERNAL_6>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2435,7 +2435,7 @@
     },
     {
       "id": 270,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>",
+      "content": "<INTERNAL_9>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2444,7 +2444,7 @@
     },
     {
       "id": 271,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>",
+      "content": "<INTERNAL_5>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2453,7 +2453,7 @@
     },
     {
       "id": 272,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>",
+      "content": "<INTERNAL_8>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2462,7 +2462,7 @@
     },
     {
       "id": 273,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>",
+      "content": "<INTERNAL_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2570,7 +2570,7 @@
     },
     {
       "id": 285,
-      "content": "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>",
+      "content": "<INTERNAL_14>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2669,7 +2669,7 @@
     },
     {
       "id": 296,
-      "content": "<CDR3_REGION>",
+      "content": "<INTERNAL_13>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2678,7 +2678,7 @@
     },
     {
       "id": 297,
-      "content": "<GENERAL_CHAIN>",
+      "content": "<INTERNAL_12>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2828,33 +2828,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 314,
-      "content": "<BBBP>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 315,
-      "content": "<FDA_APPR>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 316,
-      "content": "<HIV_ACTIVITY>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -2884,10 +2857,10 @@
       "<EOS>": 5,
       "<MOLECULAR_ENTITY>": 6,
       "<GLOBAL_INTERACTION_ATTRIBUTES>": 7,
-      "<MOLECULAR_ENTITY_ANTIGEN>": 8,
+      "<INTERNAL_0>": 8,
       "<MOLECULAR_ENTITY_EPITOPE>": 9,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>": 10,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>": 11,
+      "<INTERNAL_2>": 10,
+      "<INTERNAL_3>": 11,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CHAIN>": 12,
       "<MOLECULAR_ENTITY_TCR_BETA_VDJ>": 13,
       "<MOLECULAR_ENTITY_TCR_BETA_CDR3>": 14,
@@ -3133,9 +3106,9 @@
       "<SENTINEL_ID_197>": 254,
       "<SENTINEL_ID_198>": 255,
       "<SENTINEL_ID_199>": 256,
-      "<MOLECULAR_ENTITY_TYPE_ANTIGEN>": 257,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>": 258,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>": 259,
+      "<INTERNAL_17>": 257,
+      "<INTERNAL_15>": 258,
+      "<INTERNAL_16>": 259,
       "<ATTRIBUTE_ORGANISM>": 260,
       "<ATTRIBUTE_ORGANISM_HUMAN>": 261,
       "<ATTRIBUTE_ORGANISM_RABBIT>": 262,
@@ -3144,12 +3117,12 @@
       "<ATTRIBUTE_ORGANISM_MONKEY>": 265,
       "<ATTRIBUTE_ORGANISM_CAMEL>": 266,
       "<EPITOPE_PARATOPE_PREDICTION>": 267,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>": 268,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>": 269,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>": 270,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>": 271,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>": 272,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>": 273,
+      "<INTERNAL_7>": 268,
+      "<INTERNAL_6>": 269,
+      "<INTERNAL_9>": 270,
+      "<INTERNAL_5>": 271,
+      "<INTERNAL_8>": 272,
+      "<INTERNAL_4>": 273,
       "<MOLECULAR_ENTITY_GENERAL_PROTEIN>": 274,
       "<TIMESTEP>": 275,
       "<DIFFUSION>": 276,
@@ -3161,7 +3134,7 @@
       "<BACKSPACE>": 282,
       "<SEQUENCE_NATURAL_START>": 283,
       "<NOOP>": 284,
-      "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>": 285,
+      "<INTERNAL_14>": 285,
       "<MOLECULAR_ENTITY_SMALL_MOLECULE>": 286,
       "<MOLECULAR_ENTITY_CELL_GENE_EXPRESSION_RANKED>": 287,
       "<CELL_TYPE_CLASS>": 288,
@@ -3172,8 +3145,8 @@
       "<MOLECULAR_ENTITY_PROTEIN_CHAIN>": 293,
       "<COMPLEX_ENTITY>": 294,
       "<ALTERNATIVE>": 295,
-      "<CDR3_REGION>": 296,
-      "<GENERAL_CHAIN>": 297,
+      "<INTERNAL_13>": 296,
+      "<INTERNAL_12>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
@@ -3190,9 +3163,6 @@
       "<AUTOENCODER_LATENT_SAMPLED_Z>": 311,
       "<AUTOENCODER_TASK>": 312,
       "<DECODED_FROM_LATENT>": 313,
-      "<BBBP>": 314,
-      "<FDA_APPR>": 315,
-      "<HIV_ACTIVITY>": 316,
       "[CL:0000499]": 3522,
       "[CL:2000060]": 3523,
       "[CL:0000235]": 3524,

tokenizer/gene_tokenizer.json

@@ -77,7 +77,7 @@
     },
     {
       "id": 8,
-      "content": "<MOLECULAR_ENTITY_ANTIGEN>",
+      "content": "<INTERNAL_0>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2318,7 +2318,7 @@
     },
     {
       "id": 257,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIGEN>",
+      "content": "<INTERNAL_17>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2327,7 +2327,7 @@
     },
     {
       "id": 258,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_15>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2336,7 +2336,7 @@
     },
     {
       "id": 259,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_16>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2417,7 +2417,7 @@
     },
     {
       "id": 268,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>",
+      "content": "<INTERNAL_7>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2426,7 +2426,7 @@
     },
     {
       "id": 269,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>",
+      "content": "<INTERNAL_6>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2435,7 +2435,7 @@
     },
     {
       "id": 270,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>",
+      "content": "<INTERNAL_9>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2444,7 +2444,7 @@
     },
     {
       "id": 271,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>",
+      "content": "<INTERNAL_5>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2453,7 +2453,7 @@
     },
     {
       "id": 272,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>",
+      "content": "<INTERNAL_8>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2462,7 +2462,7 @@
     },
     {
       "id": 273,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>",
+      "content": "<INTERNAL_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2570,7 +2570,7 @@
     },
     {
       "id": 285,
-      "content": "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>",
+      "content": "<INTERNAL_14>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2669,7 +2669,7 @@
     },
     {
       "id": 296,
-      "content": "<CDR3_REGION>",
+      "content": "<INTERNAL_13>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2678,7 +2678,7 @@
     },
     {
       "id": 297,
-      "content": "<GENERAL_CHAIN>",
+      "content": "<INTERNAL_12>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2828,33 +2828,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 314,
-      "content": "<BBBP>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 315,
-      "content": "<FDA_APPR>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 316,
-      "content": "<HIV_ACTIVITY>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -2884,10 +2857,10 @@
       "<EOS>": 5,
       "<MOLECULAR_ENTITY>": 6,
       "<GLOBAL_INTERACTION_ATTRIBUTES>": 7,
-      "<MOLECULAR_ENTITY_ANTIGEN>": 8,
+      "<INTERNAL_0>": 8,
       "<MOLECULAR_ENTITY_EPITOPE>": 9,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>": 10,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>": 11,
+      "<INTERNAL_2>": 10,
+      "<INTERNAL_3>": 11,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CHAIN>": 12,
       "<MOLECULAR_ENTITY_TCR_BETA_VDJ>": 13,
       "<MOLECULAR_ENTITY_TCR_BETA_CDR3>": 14,
@@ -3133,9 +3106,9 @@
       "<SENTINEL_ID_197>": 254,
       "<SENTINEL_ID_198>": 255,
       "<SENTINEL_ID_199>": 256,
-      "<MOLECULAR_ENTITY_TYPE_ANTIGEN>": 257,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>": 258,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>": 259,
+      "<INTERNAL_17>": 257,
+      "<INTERNAL_15>": 258,
+      "<INTERNAL_16>": 259,
       "<ATTRIBUTE_ORGANISM>": 260,
       "<ATTRIBUTE_ORGANISM_HUMAN>": 261,
       "<ATTRIBUTE_ORGANISM_RABBIT>": 262,
@@ -3144,12 +3117,12 @@
       "<ATTRIBUTE_ORGANISM_MONKEY>": 265,
       "<ATTRIBUTE_ORGANISM_CAMEL>": 266,
       "<EPITOPE_PARATOPE_PREDICTION>": 267,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>": 268,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>": 269,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>": 270,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>": 271,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>": 272,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>": 273,
+      "<INTERNAL_7>": 268,
+      "<INTERNAL_6>": 269,
+      "<INTERNAL_9>": 270,
+      "<INTERNAL_5>": 271,
+      "<INTERNAL_8>": 272,
+      "<INTERNAL_4>": 273,
       "<MOLECULAR_ENTITY_GENERAL_PROTEIN>": 274,
       "<TIMESTEP>": 275,
       "<DIFFUSION>": 276,
@@ -3161,7 +3134,7 @@
       "<BACKSPACE>": 282,
       "<SEQUENCE_NATURAL_START>": 283,
       "<NOOP>": 284,
-      "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>": 285,
+      "<INTERNAL_14>": 285,
       "<MOLECULAR_ENTITY_SMALL_MOLECULE>": 286,
       "<MOLECULAR_ENTITY_CELL_GENE_EXPRESSION_RANKED>": 287,
       "<CELL_TYPE_CLASS>": 288,
@@ -3172,8 +3145,8 @@
       "<MOLECULAR_ENTITY_PROTEIN_CHAIN>": 293,
       "<COMPLEX_ENTITY>": 294,
       "<ALTERNATIVE>": 295,
-      "<CDR3_REGION>": 296,
-      "<GENERAL_CHAIN>": 297,
+      "<INTERNAL_13>": 296,
+      "<INTERNAL_12>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
@@ -3190,9 +3163,6 @@
       "<AUTOENCODER_LATENT_SAMPLED_Z>": 311,
       "<AUTOENCODER_TASK>": 312,
       "<DECODED_FROM_LATENT>": 313,
-      "<BBBP>": 314,
-      "<FDA_APPR>": 315,
-      "<HIV_ACTIVITY>": 316,
       "[100130093]": 5000,
       "[100133445]": 5001,
       "[100286793]": 5002,

tokenizer/t5_tokenizer_AA_special.json

@@ -77,7 +77,7 @@
     },
     {
       "id": 8,
-      "content": "<MOLECULAR_ENTITY_ANTIGEN>",
+      "content": "<INTERNAL_0>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_2>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_3>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2318,7 +2318,7 @@
     },
     {
       "id": 257,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIGEN>",
+      "content": "<INTERNAL_17>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2327,7 +2327,7 @@
     },
     {
       "id": 258,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>",
+      "content": "<INTERNAL_15>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2336,7 +2336,7 @@
     },
     {
       "id": 259,
-      "content": "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>",
+      "content": "<INTERNAL_16>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2417,7 +2417,7 @@
     },
     {
       "id": 268,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>",
+      "content": "<INTERNAL_7>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2426,7 +2426,7 @@
     },
     {
       "id": 269,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>",
+      "content": "<INTERNAL_6>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2435,7 +2435,7 @@
     },
     {
       "id": 270,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>",
+      "content": "<INTERNAL_9>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2444,7 +2444,7 @@
     },
     {
       "id": 271,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>",
+      "content": "<INTERNAL_5>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2453,7 +2453,7 @@
     },
     {
       "id": 272,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>",
+      "content": "<INTERNAL_8>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2462,7 +2462,7 @@
     },
     {
       "id": 273,
-      "content": "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>",
+      "content": "<INTERNAL_4>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2570,7 +2570,7 @@
     },
     {
       "id": 285,
-      "content": "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>",
+      "content": "<INTERNAL_14>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2669,7 +2669,7 @@
     },
     {
       "id": 296,
-      "content": "<CDR3_REGION>",
+      "content": "<INTERNAL_13>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2678,7 +2678,7 @@
     },
     {
       "id": 297,
-      "content": "<GENERAL_CHAIN>",
+      "content": "<INTERNAL_12>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2828,33 +2828,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 314,
-      "content": "<BBBP>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 315,
-      "content": "<FDA_APPR>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 316,
-      "content": "<HIV_ACTIVITY>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -2884,10 +2857,10 @@
       "<EOS>": 5,
       "<MOLECULAR_ENTITY>": 6,
       "<GLOBAL_INTERACTION_ATTRIBUTES>": 7,
-      "<MOLECULAR_ENTITY_ANTIGEN>": 8,
+      "<INTERNAL_0>": 8,
       "<MOLECULAR_ENTITY_EPITOPE>": 9,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN>": 10,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN>": 11,
+      "<INTERNAL_2>": 10,
+      "<INTERNAL_3>": 11,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CHAIN>": 12,
       "<MOLECULAR_ENTITY_TCR_BETA_VDJ>": 13,
       "<MOLECULAR_ENTITY_TCR_BETA_CDR3>": 14,
@@ -3133,9 +3106,9 @@
       "<SENTINEL_ID_197>": 254,
       "<SENTINEL_ID_198>": 255,
       "<SENTINEL_ID_199>": 256,
-      "<MOLECULAR_ENTITY_TYPE_ANTIGEN>": 257,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_LIGHT_CHAIN>": 258,
-      "<MOLECULAR_ENTITY_TYPE_ANTIBODY_HEAVY_CHAIN>": 259,
+      "<INTERNAL_17>": 257,
+      "<INTERNAL_15>": 258,
+      "<INTERNAL_16>": 259,
       "<ATTRIBUTE_ORGANISM>": 260,
       "<ATTRIBUTE_ORGANISM_HUMAN>": 261,
       "<ATTRIBUTE_ORGANISM_RABBIT>": 262,
@@ -3144,12 +3117,12 @@
       "<ATTRIBUTE_ORGANISM_MONKEY>": 265,
       "<ATTRIBUTE_ORGANISM_CAMEL>": 266,
       "<EPITOPE_PARATOPE_PREDICTION>": 267,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR1>": 268,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR3>": 269,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR3>": 270,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR2>": 271,
-      "<MOLECULAR_ENTITY_ANTIBODY_HEAVY_CHAIN_CDR2>": 272,
-      "<MOLECULAR_ENTITY_ANTIBODY_LIGHT_CHAIN_CDR1>": 273,
+      "<INTERNAL_7>": 268,
+      "<INTERNAL_6>": 269,
+      "<INTERNAL_9>": 270,
+      "<INTERNAL_5>": 271,
+      "<INTERNAL_8>": 272,
+      "<INTERNAL_4>": 273,
       "<MOLECULAR_ENTITY_GENERAL_PROTEIN>": 274,
       "<TIMESTEP>": 275,
       "<DIFFUSION>": 276,
@@ -3161,7 +3134,7 @@
       "<BACKSPACE>": 282,
       "<SEQUENCE_NATURAL_START>": 283,
       "<NOOP>": 284,
-      "<TARGETED_ANTIBODY_DESIGN_ENCODER_ONLY_MODE>": 285,
+      "<INTERNAL_14>": 285,
       "<MOLECULAR_ENTITY_SMALL_MOLECULE>": 286,
       "<MOLECULAR_ENTITY_CELL_GENE_EXPRESSION_RANKED>": 287,
       "<CELL_TYPE_CLASS>": 288,
@@ -3172,8 +3145,8 @@
       "<MOLECULAR_ENTITY_PROTEIN_CHAIN>": 293,
       "<COMPLEX_ENTITY>": 294,
       "<ALTERNATIVE>": 295,
-      "<CDR3_REGION>": 296,
-      "<GENERAL_CHAIN>": 297,
+      "<INTERNAL_13>": 296,
+      "<INTERNAL_12>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
       "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
@@ -3190,9 +3163,6 @@
       "<AUTOENCODER_LATENT_SAMPLED_Z>": 311,
       "<AUTOENCODER_TASK>": 312,
       "<DECODED_FROM_LATENT>": 313,
-      "<BBBP>": 314,
-      "<FDA_APPR>": 315,
-      "<HIV_ACTIVITY>": 316,
       "A": 501,
       "B": 502,
       "C": 503,