Spaces:
Build error
Build error
File size: 777 Bytes
0827183 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
class Token:
"""Represents an individual token, such as a word in an input string."""
def __init__(self, start: int, end: int, text: str, normalized: str):
"""
Parameters:
----------
start: The index of the first character of the token within the outer input string.
end: The index of the last character of the token within the outer input string.
text: The original text of the token.
normalized: A normalized version of the token. This can include things like lower casing or stemming.
"""
self.start = start
self.end = end
self.text = text
self.normalized = normalized
|