Spaces:
Build error
Build error
File size: 2,110 Bytes
64772a4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
#
# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
# Use of this file is governed by the BSD 3-clause license that
# can be found in the LICENSE.txt file in the project root.
#
#
# This default implementation of {@link TokenFactory} creates
# {@link CommonToken} objects.
#
from antlr4.Token import CommonToken
class TokenFactory(object):
pass
class CommonTokenFactory(TokenFactory):
__slots__ = 'copyText'
#
# The default {@link CommonTokenFactory} instance.
#
# <p>
# This token factory does not explicitly copy token text when constructing
# tokens.</p>
#
DEFAULT = None
def __init__(self, copyText:bool=False):
# Indicates whether {@link CommonToken#setText} should be called after
# constructing tokens to explicitly set the text. This is useful for cases
# where the input stream might not be able to provide arbitrary substrings
# of text from the input after the lexer creates a token (e.g. the
# implementation of {@link CharStream#getText} in
# {@link UnbufferedCharStream} throws an
# {@link UnsupportedOperationException}). Explicitly setting the token text
# allows {@link Token#getText} to be called at any time regardless of the
# input stream implementation.
#
# <p>
# The default value is {@code false} to avoid the performance and memory
# overhead of copying text for every token unless explicitly requested.</p>
#
self.copyText = copyText
def create(self, source, type:int, text:str, channel:int, start:int, stop:int, line:int, column:int):
t = CommonToken(source, type, channel, start, stop)
t.line = line
t.column = column
if text is not None:
t.text = text
elif self.copyText and source[1] is not None:
t.text = source[1].getText(start,stop)
return t
def createThin(self, type:int, text:str):
t = CommonToken(type=type)
t.text = text
return t
CommonTokenFactory.DEFAULT = CommonTokenFactory()
|