| | """ |
| | pygments.token |
| | ~~~~~~~~~~~~~~ |
| | |
| | Basic token types and the standard tokens. |
| | |
| | :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS. |
| | :license: BSD, see LICENSE for details. |
| | """ |
| |
|
| |
|
| | class _TokenType(tuple): |
| | parent = None |
| |
|
| | def split(self): |
| | buf = [] |
| | node = self |
| | while node is not None: |
| | buf.append(node) |
| | node = node.parent |
| | buf.reverse() |
| | return buf |
| |
|
| | def __init__(self, *args): |
| | |
| | self.subtypes = set() |
| |
|
| | def __contains__(self, val): |
| | return self is val or ( |
| | type(val) is self.__class__ and |
| | val[:len(self)] == self |
| | ) |
| |
|
| | def __getattr__(self, val): |
| | if not val or not val[0].isupper(): |
| | return tuple.__getattribute__(self, val) |
| | new = _TokenType(self + (val,)) |
| | setattr(self, val, new) |
| | self.subtypes.add(new) |
| | new.parent = self |
| | return new |
| |
|
| | def __repr__(self): |
| | return 'Token' + (self and '.' or '') + '.'.join(self) |
| |
|
| | def __copy__(self): |
| | |
| | return self |
| |
|
| | def __deepcopy__(self, memo): |
| | |
| | return self |
| |
|
| |
|
| | Token = _TokenType() |
| |
|
| | |
| | Text = Token.Text |
| | Whitespace = Text.Whitespace |
| | Escape = Token.Escape |
| | Error = Token.Error |
| | |
| | Other = Token.Other |
| |
|
| | |
| | Keyword = Token.Keyword |
| | Name = Token.Name |
| | Literal = Token.Literal |
| | String = Literal.String |
| | Number = Literal.Number |
| | Punctuation = Token.Punctuation |
| | Operator = Token.Operator |
| | Comment = Token.Comment |
| |
|
| | |
| | Generic = Token.Generic |
| |
|
| | |
| | |
| | Token.Token = Token |
| | Token.String = String |
| | Token.Number = Number |
| |
|
| |
|
| | def is_token_subtype(ttype, other): |
| | """ |
| | Return True if ``ttype`` is a subtype of ``other``. |
| | |
| | exists for backwards compatibility. use ``ttype in other`` now. |
| | """ |
| | return ttype in other |
| |
|
| |
|
| | def string_to_tokentype(s): |
| | """ |
| | Convert a string into a token type:: |
| | |
| | >>> string_to_token('String.Double') |
| | Token.Literal.String.Double |
| | >>> string_to_token('Token.Literal.Number') |
| | Token.Literal.Number |
| | >>> string_to_token('') |
| | Token |
| | |
| | Tokens that are already tokens are returned unchanged: |
| | |
| | >>> string_to_token(String) |
| | Token.Literal.String |
| | """ |
| | if isinstance(s, _TokenType): |
| | return s |
| | if not s: |
| | return Token |
| | node = Token |
| | for item in s.split('.'): |
| | node = getattr(node, item) |
| | return node |
| |
|
| |
|
| | |
| | |
| | |
| | STANDARD_TYPES = { |
| | Token: '', |
| |
|
| | Text: '', |
| | Whitespace: 'w', |
| | Escape: 'esc', |
| | Error: 'err', |
| | Other: 'x', |
| |
|
| | Keyword: 'k', |
| | Keyword.Constant: 'kc', |
| | Keyword.Declaration: 'kd', |
| | Keyword.Namespace: 'kn', |
| | Keyword.Pseudo: 'kp', |
| | Keyword.Reserved: 'kr', |
| | Keyword.Type: 'kt', |
| |
|
| | Name: 'n', |
| | Name.Attribute: 'na', |
| | Name.Builtin: 'nb', |
| | Name.Builtin.Pseudo: 'bp', |
| | Name.Class: 'nc', |
| | Name.Constant: 'no', |
| | Name.Decorator: 'nd', |
| | Name.Entity: 'ni', |
| | Name.Exception: 'ne', |
| | Name.Function: 'nf', |
| | Name.Function.Magic: 'fm', |
| | Name.Property: 'py', |
| | Name.Label: 'nl', |
| | Name.Namespace: 'nn', |
| | Name.Other: 'nx', |
| | Name.Tag: 'nt', |
| | Name.Variable: 'nv', |
| | Name.Variable.Class: 'vc', |
| | Name.Variable.Global: 'vg', |
| | Name.Variable.Instance: 'vi', |
| | Name.Variable.Magic: 'vm', |
| |
|
| | Literal: 'l', |
| | Literal.Date: 'ld', |
| |
|
| | String: 's', |
| | String.Affix: 'sa', |
| | String.Backtick: 'sb', |
| | String.Char: 'sc', |
| | String.Delimiter: 'dl', |
| | String.Doc: 'sd', |
| | String.Double: 's2', |
| | String.Escape: 'se', |
| | String.Heredoc: 'sh', |
| | String.Interpol: 'si', |
| | String.Other: 'sx', |
| | String.Regex: 'sr', |
| | String.Single: 's1', |
| | String.Symbol: 'ss', |
| |
|
| | Number: 'm', |
| | Number.Bin: 'mb', |
| | Number.Float: 'mf', |
| | Number.Hex: 'mh', |
| | Number.Integer: 'mi', |
| | Number.Integer.Long: 'il', |
| | Number.Oct: 'mo', |
| |
|
| | Operator: 'o', |
| | Operator.Word: 'ow', |
| |
|
| | Punctuation: 'p', |
| | Punctuation.Marker: 'pm', |
| |
|
| | Comment: 'c', |
| | Comment.Hashbang: 'ch', |
| | Comment.Multiline: 'cm', |
| | Comment.Preproc: 'cp', |
| | Comment.PreprocFile: 'cpf', |
| | Comment.Single: 'c1', |
| | Comment.Special: 'cs', |
| |
|
| | Generic: 'g', |
| | Generic.Deleted: 'gd', |
| | Generic.Emph: 'ge', |
| | Generic.Error: 'gr', |
| | Generic.Heading: 'gh', |
| | Generic.Inserted: 'gi', |
| | Generic.Output: 'go', |
| | Generic.Prompt: 'gp', |
| | Generic.Strong: 'gs', |
| | Generic.Subheading: 'gu', |
| | Generic.EmphStrong: 'ges', |
| | Generic.Traceback: 'gt', |
| | } |
| |
|