# # Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. # Use of this file is governed by the BSD 3-clause license that # can be found in the LICENSE.txt file in the project root. # # # This default implementation of {@link TokenFactory} creates # {@link CommonToken} objects. # from antlr4.Token import CommonToken class TokenFactory(object): pass class CommonTokenFactory(TokenFactory): # # The default {@link CommonTokenFactory} instance. # #
# This token factory does not explicitly copy token text when constructing # tokens.
# DEFAULT = None def __init__(self, copyText:bool=False): # Indicates whether {@link CommonToken#setText} should be called after # constructing tokens to explicitly set the text. This is useful for cases # where the input stream might not be able to provide arbitrary substrings # of text from the input after the lexer creates a token (e.g. the # implementation of {@link CharStream#getText} in # {@link UnbufferedCharStream} throws an # {@link UnsupportedOperationException}). Explicitly setting the token text # allows {@link Token#getText} to be called at any time regardless of the # input stream implementation. # ## The default value is {@code false} to avoid the performance and memory # overhead of copying text for every token unless explicitly requested.
# self.copyText = copyText def create(self, source, type:int, text:str, channel:int, start:int, stop:int, line:int, column:int): t = CommonToken(source, type, channel, start, stop) t.line = line t.column = column if text is not None: t.text = text elif self.copyText and source[1] is not None: t.text = source[1].getText(start,stop) return t def createThin(self, type:int, text:str): t = CommonToken(type=type) t.text = text return t CommonTokenFactory.DEFAULT = CommonTokenFactory()