File size: 161 Bytes
8d1b471 |
1 2 3 4 5 6 7 |
class CustomTokenizer:
def __init__(self):
pass
def __call__(self, text):
# No tokenization, return input text as-is
return text |
8d1b471 |
1 2 3 4 5 6 7 |
class CustomTokenizer:
def __init__(self):
pass
def __call__(self, text):
# No tokenization, return input text as-is
return text |