tokenization_nicheformer.py updated

tokenization_nicheformer.py

@@ -1,6 +1,6 @@
 from typing import List, Dict, Optional, Union, Tuple
 import numpy as np
-from transformers import PreTrainedTokenizer, AutoTokenizer
+from transformers import PreTrainedTokenizer
 from dataclasses import dataclass
 import torch
 import anndata as ad
@@ -405,7 +405,4 @@ class NicheformerTokenizer(PreTrainedTokenizer):
     def get_special_tokens_mask(self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False) -> List[int]:
         """Get list where entries are [1] if a token is [special] else [0]."""
         # Consider tokens < aux_tokens as special
-        return [1 if token_id < self.aux_tokens else 0 for token_id in token_ids_0] 
-
-# Register the tokenizer class
-AutoTokenizer.register("nicheformer", "tokenization_nicheformer.NicheformerTokenizer") 
+        return [1 if token_id < self.aux_tokens else 0 for token_id in token_ids_0]