Repository: pygments/pygments

Issue:
[feature] Support for PDDL programming language #381 
I'm recently working with [PDDL programming language](https://en.wikipedia.org/wiki/Planning_Domain_Definition_Language) -- could we get support for fancy syntax highlighting?

Open to help here!

Discussion:
@jeanas: Consider contributing the lexer. Steps are outlined on https://pygments.org/docs/lexerdevelopment/

The maintainers of Pygments can't know every language on Earth, so we mostly rely on “external” contributions to add new lexers.

Solution patch:
diff --git a/doc/docs/contributing.rst b/doc/docs/contributing.rst
index c91d9f6a..e38069c5 100644
--- a/doc/docs/contributing.rst
+++ b/doc/docs/contributing.rst
@@ -39,12 +39,12 @@ Python support
 --------------
 
 Pygments supports all supported Python versions as per the `Python
-Developer's Guide <devguide>`_. Additionally, the default Python
+Developer's Guide`_. Additionally, the default Python
 version of the latest stable version of RHEL, Ubuntu LTS, and Debian
 are supported, even if they're officially EOL. Supporting other
 end-of-life versions is a non-goal of Pygments.
 
-.. _devguide: https://devguide.python.org/versions/
+.. _Python Developer's Guide: https://devguide.python.org/versions/
 
 
 Validation
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index 3330c1d5..78eeed13 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -44,7 +44,15 @@ def webify(color):
     if color.startswith('calc') or color.startswith('var'):
         return color
     else:
-        return '#' + color
+        # Check if the color can be shortened from 6 to 3 characters
+        color = color.upper()
+        if (len(color) == 6 and
+            (    color[0] == color[1]
+             and color[2] == color[3]
+             and color[4] == color[5])):
+            return f'#{color[0]}{color[2]}{color[4]}'
+        else:
+            return f'#{color}'
 
 
 def _get_ttype_class(ttype):
diff --git a/pygments/lexers/_googlesql_builtins.py b/pygments/lexers/_googlesql_builtins.py
new file mode 100644
index 00000000..b591546c
--- /dev/null
+++ b/pygments/lexers/_googlesql_builtins.py
@@ -0,0 +1,918 @@
+"""
+    pygments.lexers._googlesql_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Autogenerated data files for the GoogleSQL lexer.
+
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+constants = [
+    'FALSE',
+    'NULL',
+    'TRUE',
+    'UNKNOWN',
+]
+
+# Everything below this line is auto-generated from the GoogleSQL source code.
+# ----------------------------------------------------------------------------
+
+functionnames = [
+    'ABS',
+    'ACOS',
+    'ACOSH',
+    'AEAD.DECRYPT_BYTES',
+    'AEAD.DECRYPT_STRING',
+    'AEAD.ENCRYPT',
+    'AEAD.ENVELOPE_DECRYPT_BYTES',
+    'AEAD.ENVELOPE_DECRYPT_STRING',
+    'AEAD.ENVELOPE_ENCRYPT',
+    'ALL_DIFFERENT',
+    'ANON_AVG',
+    'ANON_COUNT',
+    'ANON_COUNT',
+    'ANON_PERCENTILE_CONT',
+    'ANON_QUANTILES',
+    'ANON_STDDEV_POP',
+    'ANON_SUM',
+    'ANON_VAR_POP',
+    'ANY_VALUE',
+    'APPROX_COSINE_DISTANCE',
+    'APPROX_COUNT_DISTINCT',
+    'APPROX_DOT_PRODUCT',
+    'APPROX_EUCLIDEAN_DISTANCE',
+    'APPROX_QUANTILES',
+    'APPROX_TOP_COUNT',
+    'APPROX_TOP_SUM',
+    'ARRAY[KEY()]',
+    'ARRAY[SAFE_KEY()]',
+    'ARRAY_AGG',
+    'ARRAY_AVG',
+    'ARRAY_CONCAT',
+    'ARRAY_CONCAT_AGG',
+    'ARRAY_FILTER',
+    'ARRAY_FIND',
+    'ARRAY_FIND_ALL',
+    'ARRAY_FIRST',
+    'ARRAY_FIRST_N',
+    'ARRAY_INCLUDES',
+    'ARRAY_INCLUDES_ALL',
+    'ARRAY_INCLUDES_ANY',
+    'ARRAY_IS_DISTINCT',
+    'ARRAY_LAST',
+    'ARRAY_LAST_N',
+    'ARRAY_LENGTH',
+    'ARRAY_MAX',
+    'ARRAY_MIN',
+    'ARRAY_OFFSET',
+    'ARRAY_OFFSETS',
+    'ARRAY_REMOVE_FIRST_N',
+    'ARRAY_REMOVE_LAST_N',
+    'ARRAY_REVERSE',
+    'ARRAY_SLICE',
+    'ARRAY_SUM',
+    'ARRAY_TO_STRING',
+    'ARRAY_TRANSFORM',
+    'ARRAY_ZIP',
+    'ASCII',
+    'ASIN',
+    'ASINH',
+    'ATAN',
+    'ATAN2',
+    'ATANH',
+    'AVG',
+    'BIT_AND',
+    'BIT_COUNT',
+    'BIT_OR',
+    'BIT_XOR',
+    'BOOL',
+    'BOOL_ARRAY',
+    'BYTE_LENGTH',
+    'CASE',
+    'CAST',
+    'CBRT',
+    'CEIL',
+    'CEILING',
+    'CHARACTER_LENGTH',
+    'CHAR_LENGTH',
+    'CHR',
+    'COALESCE',
+    'CODE_POINTS_TO_BYTES',
+    'CODE_POINTS_TO_STRING',
+    'COLLATE',
+    'CONCAT',
+    'CORR',
+    'COS',
+    'COSH',
+    'COSINE_DISTANCE',
+    'COT',
+    'COTH',
+    'COUNT',
+    'COUNT(*)',
+    'COUNTIF',
+    'COVAR_POP',
+    'COVAR_SAMP',
+    'CSC',
+    'CSCH',
+    'CUME_DIST',
+    'CURRENT_DATE',
+    'CURRENT_DATETIME',
+    'CURRENT_TIME',
+    'CURRENT_TIMESTAMP',
+    'D3A_COUNT.EXTRACT',
+    'D3A_COUNT.INIT',
+    'D3A_COUNT.MERGE',
+    'D3A_COUNT.MERGE_PARTIAL',
+    'D3A_COUNT.TO_HLL',
+    'DATE',
+    'DATETIME',
+    'DATETIME_ADD',
+    'DATETIME_BUCKET',
+    'DATETIME_DIFF',
+    'DATETIME_SUB',
+    'DATETIME_TRUNC',
+    'DATE_ADD',
+    'DATE_BUCKET',
+    'DATE_DIFF',
+    'DATE_FROM_UNIX_DATE',
+    'DATE_SUB',
+    'DATE_TRUNC',
+    'DENSE_RANK',
+    'DESTINATION_NODE_ID',
+    'DETERMINISTIC_DECRYPT_BYTES',
+    'DETERMINISTIC_DECRYPT_STRING',
+    'DETERMINISTIC_ENCRYPT',
+    'DIV',
+    'DOT_PRODUCT',
+    'EDGES',
+    'EDIT_DISTANCE',
+    'ELEMENTWISE_AVG',
+    'ELEMENTWISE_SUM',
+    'ELEMENT_DEFINITION_NAME',
+    'ELEMENT_ID',
+    'ENDS_WITH',
+    'ENUM_VALUE_DESCRIPTOR_PROTO',
+    'ERROR',
+    'EUCLIDEAN_DISTANCE',
+    'EXP',
+    'EXTRACT',
+    'EXTRACT_FOR_DP_APPROX_COUNT_DISTINCT',
+    'FARM_FINGERPRINT',
+    'FILTER_FIELDS',
+    'FIRST_VALUE',
+    'FLATTEN',
+    'FLOAT32',
+    'FLOAT32_ARRAY',
+    'FLOAT64',
+    'FLOAT64_ARRAY',
+    'FLOOR',
+    'FORMAT',
+    'FORMAT_DATE',
+    'FORMAT_DATETIME',
+    'FORMAT_TIME',
+    'FORMAT_TIMESTAMP',
+    'FROM_BASE32',
+    'FROM_BASE64',
+    'FROM_HEX',
+    'GENERATE_ARRAY',
+    'GENERATE_DATE_ARRAY',
+    'GENERATE_RANGE_ARRAY',
+    'GENERATE_TIMESTAMP_ARRAY',
+    'GENERATE_UUID',
+    'GREATEST',
+    'GROUPING',
+    'HLL_COUNT.EXTRACT',
+    'HLL_COUNT.INIT',
+    'HLL_COUNT.MERGE',
+    'HLL_COUNT.MERGE_PARTIAL',
+    'IEEE_DIVIDE',
+    'IF',
+    'IFERROR',
+    'IFNULL',
+    'IN UNNEST',
+    'INITCAP',
+    'INIT_FOR_DP_APPROX_COUNT_DISTINCT',
+    'INSTR',
+    'INT64',
+    'INT64_ARRAY',
+    'IS DESTINATION OF',
+    'IS DISTINCT FROM',
+    'IS NOT DISTINCT FROM',
+    'IS SOURCE OF',
+    'ISERROR',
+    'IS_ACYCLIC',
+    'IS_INF',
+    'IS_NAN',
+    'IS_SIMPLE',
+    'IS_TRAIL',
+    'JSON_ARRAY',
+    'JSON_ARRAY_APPEND',
+    'JSON_ARRAY_INSERT',
+    'JSON_CONTAINS',
+    'JSON_EXTRACT',
+    'JSON_EXTRACT_ARRAY',
+    'JSON_EXTRACT_SCALAR',
+    'JSON_EXTRACT_STRING_ARRAY',
+    'JSON_KEYS',
+    'JSON_OBJECT',
+    'JSON_QUERY',
+    'JSON_QUERY_ARRAY',
+    'JSON_REMOVE',
+    'JSON_SET',
+    'JSON_STRIP_NULLS',
+    'JSON_TYPE',
+    'JSON_VALUE',
+    'JSON_VALUE_ARRAY',
+    'JUSTIFY_DAYS',
+    'JUSTIFY_HOURS',
+    'JUSTIFY_INTERVAL',
+    'KEYS.ADD_KEY_FROM_RAW_BYTES',
+    'KEYS.KEYSET_CHAIN',
+    'KEYS.KEYSET_FROM_JSON',
+    'KEYS.KEYSET_LENGTH',
+    'KEYS.KEYSET_TO_JSON',
+    'KEYS.NEW_KEYSET',
+    'KEYS.NEW_WRAPPED_KEYSET',
+    'KEYS.REWRAP_KEYSET',
+    'KEYS.ROTATE_KEYSET',
+    'KEYS.ROTATE_WRAPPED_KEYSET',
+    'KLL_QUANTILES.EXTRACT_FLOAT64',
+    'KLL_QUANTILES.EXTRACT_INT64',
+    'KLL_QUANTILES.EXTRACT_POINT_FLOAT64',
+    'KLL_QUANTILES.EXTRACT_POINT_INT64',
+    'KLL_QUANTILES.INIT_FLOAT64',
+    'KLL_QUANTILES.INIT_INT64',
+    'KLL_QUANTILES.MERGE_FLOAT64',
+    'KLL_QUANTILES.MERGE_INT64',
+    'KLL_QUANTILES.MERGE_PARTIAL',
+    'KLL_QUANTILES.MERGE_POINT_FLOAT64',
+    'KLL_QUANTILES.MERGE_POINT_INT64',
+    'L1_NORM',
+    'L2_NORM',
+    'LABELS',
+    'LAG',
+    'LAST_DAY',
+    'LAST_VALUE',
+    'LAX_BOOL',
+    'LAX_BOOL_ARRAY',
+    'LAX_FLOAT32',
+    'LAX_FLOAT32_ARRAY',
+    'LAX_FLOAT64',
+    'LAX_FLOAT64_ARRAY',
+    'LAX_INT64',
+    'LAX_INT64_ARRAY',
+    'LAX_STRING',
+    'LAX_STRING_ARRAY',
+    'LEAD',
+    'LEAST',
+    'LEFT',
+    'LENGTH',
+    'LIKE ALL',
+    'LIKE ALL UNNEST',
+    'LIKE ANY',
+    'LIKE ANY UNNEST',
+    'LN',
+    'LOG',
+    'LOG10',
+    'LOGICAL_AND',
+    'LOGICAL_OR',
+    'LOWER',
+    'LPAD',
+    'LTRIM',
+    'MAKE_INTERVAL',
+    'MANHATTAN_DISTANCE',
+    'MAP_CARDINALITY',
+    'MAP_CONTAINS_KEY',
+    'MAP_DELETE',
+    'MAP_EMPTY',
+    'MAP_ENTRIES_SORTED',
+    'MAP_ENTRIES_UNSORTED',
+    'MAP_FILTER',
+    'MAP_FROM_ARRAY',
+    'MAP_GET',
+    'MAP_INSERT',
+    'MAP_INSERT_OR_REPLACE',
+    'MAP_KEYS_SORTED',
+    'MAP_KEYS_UNSORTED',
+    'MAP_REPLACE',
+    'MAP_VALUES_SORTED',
+    'MAP_VALUES_SORTED_BY_KEY',
+    'MAP_VALUES_UNSORTED',
+    'MAX',
+    'MD5',
+    'MERGE_PARTIAL_FOR_DP_APPROX_COUNT_DISTINCT',
+    'MIN',
+    'MOD',
+    'NET.HOST',
+    'NET.IPV4_FROM_INT64',
+    'NET.IPV4_TO_INT64',
+    'NET.IP_FROM_STRING',
+    'NET.IP_NET_MASK',
+    'NET.IP_TO_STRING',
+    'NET.IP_TRUNC',
+    'NET.PUBLIC_SUFFIX',
+    'NET.REG_DOMAIN',
+    'NET.SAFE_IP_FROM_STRING',
+    'NEW_UUID',
+    'NODES',
+    'NORMALIZE',
+    'NORMALIZE_AND_CASEFOLD',
+    'NOT LIKE ALL',
+    'NOT LIKE ALL UNNEST',
+    'NOT LIKE ANY',
+    'NOT LIKE ANY UNNEST',
+    'NTH_VALUE',
+    'NTILE',
+    'NULLIF',
+    'NULLIFERROR',
+    'NULLIFZERO',
+    'OCTET_LENGTH',
+    'OFFSET',
+    'ORDINAL',
+    'PARSE_BIGNUMERIC',
+    'PARSE_DATE',
+    'PARSE_DATETIME',
+    'PARSE_JSON',
+    'PARSE_NUMERIC',
+    'PARSE_TIME',
+    'PARSE_TIMESTAMP',
+    'PATH',
+    'PATH_FIRST',
+    'PATH_LAST',
+    'PATH_LENGTH',
+    'PERCENTILE_CONT',
+    'PERCENTILE_DISC',
+    'PERCENT_RANK',
+    'PI',
+    'PIVOT',
+    'PI_BIGNUMERIC',
+    'PI_NUMERIC',
+    'POW',
+    'POWER',
+    'PROPERTY_EXISTS',
+    'PROPERTY_NAMES',
+    'PROTO_MAP_CONTAINS_KEY',
+    'PROTO_MODIFY_MAP',
+    'RAND',
+    'RANGE',
+    'RANGE_BUCKET',
+    'RANGE_CONTAINS',
+    'RANGE_END',
+    'RANGE_INTERSECT',
+    'RANGE_IS_END_UNBOUNDED',
+    'RANGE_IS_START_UNBOUNDED',
+    'RANGE_OVERLAPS',
+    'RANGE_START',
+    'RANK',
+    'REGEXP_CONTAINS',
+    'REGEXP_EXTRACT',
+    'REGEXP_EXTRACT_ALL',
+    'REGEXP_INSTR',
+    'REGEXP_REPLACE',
+    'REGEXP_SUBSTR',
+    'REPEAT',
+    'REPLACE',
+    'REVERSE',
+    'RIGHT',
+    'ROUND',
+    'ROW_NUMBER',
+    'RPAD',
+    'RTRIM',
+    'S2_CELLIDFROMPOINT',
+    'S2_COVERINGCELLIDS',
+    'SAFE_ADD',
+    'SAFE_CONVERT_BYTES_TO_STRING',
+    'SAFE_DIVIDE',
+    'SAFE_MULTIPLY',
+    'SAFE_NEGATE',
+    'SAFE_OFFSET',
+    'SAFE_ORDINAL',
+    'SAFE_SUBTRACT',
+    'SAFE_TO_JSON',
+    'SAME',
+    'SEC',
+    'SECH',
+    'SESSION_USER',
+    'SHA1',
+    'SHA256',
+    'SHA512',
+    'SIGN',
+    'SIN',
+    'SINH',
+    'SOUNDEX',
+    'SOURCE_NODE_ID',
+    'SPLIT',
+    'SPLIT_SUBSTR',
+    'SQRT',
+    'STARTS_WITH',
+    'STDDEV',
+    'STDDEV_POP',
+    'STDDEV_SAMP',
+    'STRING',
+    'STRING_AGG',
+    'STRING_ARRAY',
+    'STRPOS',
+    'ST_ANGLE',
+    'ST_AREA',
+    'ST_ASBINARY',
+    'ST_ASGEOJSON',
+    'ST_ASKML',
+    'ST_ASTEXT',
+    'ST_AZIMUTH',
+    'ST_BOUNDARY',
+    'ST_BOUNDINGBOX',
+    'ST_BUFFER',
+    'ST_BUFFERWITHTOLERANCE',
+    'ST_CENTROID',
+    'ST_CENTROID_AGG',
+    'ST_CLOSESTPOINT',
+    'ST_CLUSTERDBSCAN',
+    'ST_CONTAINS',
+    'ST_CONVEXHULL',
+    'ST_COVEREDBY',
+    'ST_COVERS',
+    'ST_DIFFERENCE',
+    'ST_DIMENSION',
+    'ST_DISJOINT',
+    'ST_DISTANCE',
+    'ST_DUMP',
+    'ST_DUMPPOINTS',
+    'ST_DWITHIN',
+    'ST_ENDPOINT',
+    'ST_EQUALS',
+    'ST_EXTENT',
+    'ST_EXTERIORRING',
+    'ST_GEOGFROM',
+    'ST_GEOGFROMGEOJSON',
+    'ST_GEOGFROMKML',
+    'ST_GEOGFROMTEXT',
+    'ST_GEOGFROMWKB',
+    'ST_GEOGPOINT',
+    'ST_GEOGPOINTFROMGEOHASH',
+    'ST_GEOHASH',
+    'ST_GEOMETRYTYPE',
+    'ST_HAUSDORFFDISTANCE',
+    'ST_HAUSDORFFDWITHIN',
+    'ST_INTERIORRINGS',
+    'ST_INTERSECTION',
+    'ST_INTERSECTS',
+    'ST_INTERSECTSBOX',
+    'ST_ISCLOSED',
+    'ST_ISCOLLECTION',
+    'ST_ISEMPTY',
+    'ST_ISRING',
+    'ST_LENGTH',
+    'ST_LINEINTERPOLATEPOINT',
+    'ST_LINELOCATEPOINT',
+    'ST_LINESUBSTRING',
+    'ST_MAKELINE',
+    'ST_MAKEPOLYGON',
+    'ST_MAKEPOLYGONORIENTED',
+    'ST_MAXDISTANCE',
+    'ST_NEAREST_NEIGHBORS',
+    'ST_NPOINTS',
+    'ST_NUMGEOMETRIES',
+    'ST_NUMPOINTS',
+    'ST_PERIMETER',
+    'ST_POINTN',
+    'ST_SIMPLIFY',
+    'ST_SNAPTOGRID',
+    'ST_STARTPOINT',
+    'ST_TOUCHES',
+    'ST_UNARYUNION',
+    'ST_UNION',
+    'ST_UNION_AGG',
+    'ST_WITHIN',
+    'ST_X',
+    'ST_Y',
+    'SUBSTR',
+    'SUBSTRING',
+    'SUM',
+    'TAN',
+    'TANH',
+    'TIME',
+    'TIMESTAMP',
+    'TIMESTAMP_ADD',
+    'TIMESTAMP_BUCKET',
+    'TIMESTAMP_DIFF',
+    'TIMESTAMP_FROM_UNIX_MICROS',
+    'TIMESTAMP_FROM_UNIX_MILLIS',
+    'TIMESTAMP_FROM_UNIX_SECONDS',
+    'TIMESTAMP_MICROS',
+    'TIMESTAMP_MILLIS',
+    'TIMESTAMP_SECONDS',
+    'TIMESTAMP_SUB',
+    'TIMESTAMP_TRUNC',
+    'TIME_ADD',
+    'TIME_DIFF',
+    'TIME_SUB',
+    'TIME_TRUNC',
+    'TO_BASE32',
+    'TO_BASE64',
+    'TO_CODE_POINTS',
+    'TO_HEX',
+    'TO_JSON',
+    'TO_JSON_STRING',
+    'TRANSLATE',
+    'TRIM',
+    'TRUNC',
+    'TYPEOF',
+    'UNICODE',
+    'UNIX_DATE',
+    'UNIX_MICROS',
+    'UNIX_MILLIS',
+    'UNIX_SECONDS',
+    'UNNEST',
+    'UNPIVOT',
+    'UPPER',
+    'VARIANCE',
+    'VAR_POP',
+    'VAR_SAMP',
+    'ZEROIFNULL',
+]
+
+keywords = [
+    'ABORT',
+    'ACCESS',
+    'ACTION',
+    'ACYCLIC',
+    'ADD',
+    'AFTER',
+    'AGGREGATE',
+    'ALL',
+    'ALTER',
+    'ALWAYS',
+    'ANALYZE',
+    'AND',
+    'ANY',
+    'APPROX',
+    'ARE',
+    'AS',
+    'ASC',
+    'ASCENDING',
+    'ASSERT',
+    'ASSERT_ROWS_MODIFIED',
+    'AT',
+    'BATCH',
+    'BEGIN',
+    'BETWEEN',
+    'BIGDECIMAL',
+    'BREAK',
+    'BY',
+    'CALL',
+    'CASCADE',
+    'CASE',
+    'CAST',
+    'CHECK',
+    'CLAMPED',
+    'CLONE',
+    'CLUSTER',
+    'COLLATE',
+    'COLUMN',
+    'COLUMNS',
+    'COMMIT',
+    'CONFLICT',
+    'CONNECTION',
+    'CONSTANT',
+    'CONSTRAINT',
+    'CONTAINS',
+    'CONTINUE',
+    'COPY',
+    'CORRESPONDING',
+    'CREATE',
+    'CROSS',
+    'CUBE',
+    'CURRENT',
+    'CYCLE',
+    'DATA',
+    'DATABASE',
+    'DAY',
+    'DAYOFWEEK',
+    'DAYOFYEAR',
+    'DECIMAL',
+    'DECLARE',
+    'DEFAULT',
+    'DEFINE',
+    'DEFINER',
+    'DELETE',
+    'DELETION',
+    'DEPTH',
+    'DESC',
+    'DESCENDING',
+    'DESCRIBE',
+    'DESCRIPTOR',
+    'DESTINATION',
+    'DETERMINISTIC',
+    'DISTINCT',
+    'DO',
+    'DROP',
+    'EDGE',
+    'ELSE',
+    'ELSEIF',
+    'END',
+    'ENFORCED',
+    'ERROR',
+    'ESCAPE',
+    'EXCEPT',
+    'EXCEPTION',
+    'EXCLUDE',
+    'EXECUTE',
+    'EXISTS',
+    'EXPLAIN',
+    'EXPORT',
+    'EXTEND',
+    'EXTERNAL',
+    'EXTRACT',
+    'FALSE',
+    'FETCH',
+    'FIELD',
+    'FILES',
+    'FILL',
+    'FILTER',
+    'FIRST',
+    'FOLLOWING',
+    'FOR',
+    'FOREIGN',
+    'FORK',
+    'FORMAT',
+    'FRIDAY',
+    'FROM',
+    'FULL',
+    'FUNCTION',
+    'GENERATED',
+    'GRANT',
+    'GRAPH',
+    'GRAPH_TABLE',
+    'GROUP',
+    'GROUPING',
+    'GROUPS',
+    'GROUP_ROWS',
+    'HAS',
+    'HASH',
+    'HAVING',
+    'HIDDEN',
+    'HOUR',
+    'IDENTITY',
+    'IF',
+    'IGNORE',
+    'IMMEDIATE',
+    'IMMUTABLE',
+    'IMPORT',
+    'IN',
+    'INCLUDE',
+    'INCREMENT',
+    'INDEX',
+    'INNER',
+    'INOUT',
+    'INPUT',
+    'INSERT',
+    'INTERLEAVE',
+    'INTERSECT',
+    'INTO',
+    'INVOKER',
+    'IS',
+    'ISOLATION',
+    'ISOWEEK ',
+    'ISOYEAR',
+    'ITERATE',
+    'JOIN',
+    'KEY',
+    'LABEL',
+    'LABELED',
+    'LANGUAGE',
+    'LAST',
+    'LATERAL',
+    'LEAVE',
+    'LEFT',
+    'LET',
+    'LEVEL',
+    'LIKE',
+    'LIMIT',
+    'LOAD',
+    'LOG',
+    'LOOKUP',
+    'LOOP',
+    'MACRO',
+    'MATCH',
+    'MATCHED',
+    'MATCH_RECOGNIZE',
+    'MATERIALIZED',
+    'MAX',
+    'MAXVALUE',
+    'MEASURES',
+    'MERGE',
+    'MESSAGE',
+    'METADATA',
+    'MICROSECOND',
+    'MILLISECOND',
+    'MIN',
+    'MINUTE',
+    'MINVALUE',
+    'MODEL',
+    'MODULE',
+    'MONDAY',
+    'MONTH',
+    'NAME',
+    'NANOSECOND',
+    'NATURAL',
+    'NEW',
+    'NEXT',
+    'NO',
+    'NODE',
+    'NOT',
+    'NOTHING',
+    'NULL',
+    'NULLS',
+    'NULL_FILTERED',
+    'OF',
+    'OFFSET',
+    'ON',
+    'ONEOF_CASE',
+    'ONLY',
+    'OPTIONAL',
+    'OPTIONS',
+    'OR',
+    'ORDER',
+    'OUT',
+    'OUTER',
+    'OUTPUT',
+    'OVER',
+    'OVERWRITE',
+    'PARENT',
+    'PARTITION',
+    'PARTITIONS',
+    'PAST',
+    'PATH',
+    'PATHS',
+    'PATTERN',
+    'PERCENT',
+    'PIVOT',
+    'POLICIES',
+    'POLICY',
+    'PRECEDING',
+    'PRIMARY',
+    'PRIVATE',
+    'PRIVILEGE',
+    'PRIVILEGES',
+    'PROCEDURE',
+    'PROJECT',
+    'PROPERTIES',
+    'PROPERTY',
+    'PUBLIC',
+    'QUALIFY',
+    'QUARTER',
+    'RAISE',
+    'RAW',
+    'READ',
+    'RECURSIVE',
+    'REFERENCES',
+    'REMOTE',
+    'REMOVE',
+    'RENAME',
+    'REPEAT',
+    'REPEATABLE',
+    'REPLACE',
+    'REPLACE_FIELDS',
+    'REPLICA',
+    'REPORT',
+    'RESPECT',
+    'RESTRICT',
+    'RESTRICTION',
+    'RETURN',
+    'RETURNS',
+    'REVOKE',
+    'RIGHT',
+    'ROLLBACK',
+    'ROLLUP',
+    'ROW',
+    'ROWS',
+    'RUN',
+    'SAFE_CAST',
+    'SATURDAY',
+    'SCHEMA',
+    'SEARCH',
+    'SECOND ',
+    'SECURITY',
+    'SELECT',
+    'SEQUENCE',
+    'SET',
+    'SETS',
+    'SHORTEST',
+    'SHOW',
+    'SIMPLE',
+    'SKIP',
+    'SNAPSHOT',
+    'SOME',
+    'SOURCE',
+    'SQL',
+    'STABLE',
+    'START',
+    'STATIC_DESCRIBE',
+    'STORED',
+    'STORING',
+    'STRICT',
+    'SUNDAY',
+    'SYSTEM',
+    'SYSTEM_TIME',
+    'TABLE',
+    'TABLES',
+    'TABLESAMPLE',
+    'TARGET',
+    'TEMP',
+    'TEMPORARY',
+    'THEN',
+    'THURSDAY',
+    'TO',
+    'TRAIL',
+    'TRANSACTION',
+    'TRANSFORM',
+    'TREAT',
+    'TRUE',
+    'TRUNCATE',
+    'TUESDAY',
+    'TYPE',
+    'UNBOUNDED',
+    'UNDROP',
+    'UNION',
+    'UNIQUE',
+    'UNKNOWN',
+    'UNNEST',
+    'UNPIVOT',
+    'UNTIL',
+    'UPDATE',
+    'USING',
+    'VALUE',
+    'VALUES',
+    'VECTOR',
+    'VIEW',
+    'VIEWS',
+    'VOLATILE',
+    'WALK',
+    'WEDNESDAY',
+    'WEEK',
+    'WEIGHT',
+    'WHEN',
+    'WHERE',
+    'WHILE',
+    'WINDOW',
+    'WITH',
+    'WITHIN',
+    'WRITE',
+    'YEAR',
+    'ZONE',
+]
+
+operators = [
+    '!=',
+    '&',
+    '*',
+    '+',
+    '-',
+    '/',
+    '<',
+    '<<',
+    '<=',
+    '=',
+    '>',
+    '>=',
+    '>>',
+    '^',
+    '|',
+    '||',
+    '~',
+]
+
+types = [
+    'ARRAY',
+    'BIGNUMERIC',
+    'BOOL',
+    'BYTES',
+    'DATE',
+    'DATETIME',
+    'DOUBLE',
+    'ENUM',
+    'EXTENDED',
+    'FLOAT',
+    'GEOGRAPHY',
+    'GRAPH_ELEMENT',
+    'GRAPH_PATH',
+    'INT32',
+    'INT64',
+    'INTERVAL',
+    'JSON',
+    'MAP',
+    'MEASURE',
+    'NUMERIC',
+    'PROTO',
+    'RANGE',
+    'STRING',
+    'STRUCT',
+    'TIME',
+    'TIMESTAMP',
+    'TIMESTAMP_PICOS',
+    'TOKENLIST',
+    'UINT32',
+    'UINT64',
+    'UUID',
+]
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 69a1cb18..7395e8cc 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -195,6 +195,7 @@ LEXERS = {
     'GoLexer': ('pygments.lexers.go', 'Go', ('go', 'golang'), ('*.go',), ('text/x-gosrc',)),
     'GoloLexer': ('pygments.lexers.jvm', 'Golo', ('golo',), ('*.golo',), ()),
     'GoodDataCLLexer': ('pygments.lexers.business', 'GoodData-CL', ('gooddata-cl',), ('*.gdc',), ('text/x-gooddata-cl',)),
+    'GoogleSqlLexer': ('pygments.lexers.sql', 'GoogleSQL', ('googlesql', 'zetasql'), ('*.googlesql', '*.googlesql.sql'), ('text/x-google-sql', 'text/x-google-sql-aux')),
     'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
     'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
     'GraphQLLexer': ('pygments.lexers.graphql', 'GraphQL', ('graphql',), ('*.graphql',), ()),
@@ -375,6 +376,7 @@ LEXERS = {
     'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),
     'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)),
     'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)),
+    'PddlLexer': ('pygments.lexers.pddl', 'PDDL', ('pddl',), ('*.pddl',), ()),
     'PegLexer': ('pygments.lexers.grammar_notation', 'PEG', ('peg',), ('*.peg',), ('text/x-peg',)),
     'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6', 'raku'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod', '*.rakutest', '*.rakudoc'), ('text/x-perl6', 'application/x-perl6')),
     'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t', '*.perl'), ('text/x-perl', 'application/x-perl')),
@@ -437,6 +439,7 @@ LEXERS = {
     'RedLexer': ('pygments.lexers.rebol', 'Red', ('red', 'red/system'), ('*.red', '*.reds'), ('text/x-red', 'text/x-red-system')),
     'RedcodeLexer': ('pygments.lexers.esoteric', 'Redcode', ('redcode',), ('*.cw',), ()),
     'RegeditLexer': ('pygments.lexers.configs', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)),
+    'RegoLexer': ('pygments.lexers.rego', 'Rego', ('rego',), ('*.rego',), ('text/x-rego',)),
     'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resourcebundle', 'resource'), (), ()),
     'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),
     'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
diff --git a/pygments/lexers/bqn.py b/pygments/lexers/bqn.py
index b0794ee0..0a815a04 100644
--- a/pygments/lexers/bqn.py
+++ b/pygments/lexers/bqn.py
@@ -26,6 +26,10 @@ class BQNLexer(RegexLexer):
     mimetypes = []
     version_added = '2.16'
 
+    # An inter_word_char. Necessary because \w matches all alphanumeric
+    # Unicode characters, including ones (e.g., 𝕊) that BQN treats special.
+    _iwc = r'((?=[^𝕎𝕏𝔽𝔾𝕊𝕨𝕩𝕗𝕘𝕤𝕣])\w)'
+
     tokens = {
         'root': [
             # Whitespace
@@ -57,34 +61,35 @@ class BQNLexer(RegexLexer):
             # ===================
             # Since this token type is important in BQN, it is not included in
             # the punctuation token type but rather in the following one
-            (r'[\(\)]', String.Regex), 
+            (r'[\(\)]', String.Regex),
             #
             # Numbers
             # =======
             # Includes the numeric literals and the Nothing character
-            (r'¯?([0-9]+\.?[0-9]+|[0-9]+)([Ee][¯]?[0-9]+)?|¯|∞|π|·', Number),
+            (r'¯?[0-9](([0-9]|_)*\.?([0-9]|_)+|([0-9]|_)*)([Ee][¯]?([0-9]|_)+)?|¯|∞|π|·', Number),
             #
             # Variables
             # =========
-            (r'\b[a-z]\w*\b', Name.Variable),
-            #
-            # 1-Modifiers
-            # ===========
-            (r'[˙˜˘¨⌜⁼´˝`𝕣]', Name.Attribute),
-            (r'\b_[a-zA-Z0-9]+\b', Name.Attribute),
+            (r'[a-z]' + _iwc + r'*', Name.Variable),
             #
             # 2-Modifiers
             # ===========
+            # Needs to come before the 1-modifiers due to _𝕣 and _𝕣_
             (r'[∘○⊸⟜⌾⊘◶⎉⚇⍟⎊]', Name.Property),
-            (r'\b_[a-zA-Z0-9]+_\b', Name.Property),
+            (r'_(𝕣|[a-zA-Z0-9]+)_', Name.Property),
+            #
+            # 1-Modifiers
+            # ===========
+            (r'[˙˜˘¨⌜⁼´˝`𝕣]', Name.Attribute),
+            (r'_(𝕣|[a-zA-Z0-9]+)', Name.Attribute),
             #
             # Functions
             # =========
             # The monadic or dyadic function primitives and function
             # operands and arguments, along with function self-reference
-            (r'[+\-×÷\*√⌊⌈∧∨¬|≤<>≥=≠≡≢⊣⊢⥊∾≍⋈↑↓↕«»⌽⍉/⍋⍒⊏⊑⊐⊒∊⍷⊔!𝕎𝕏𝔽𝔾𝕊]',
+            (r'[+\-×÷\⋆√⌊⌈∧∨¬|≤<>≥=≠≡≢⊣⊢⥊∾≍⋈↑↓↕«»⌽⍉/⍋⍒⊏⊑⊐⊒∊⍷⊔!𝕎𝕏𝔽𝔾𝕊]',
              Operator),
-            (r'[A-Z]\w*|•\w+\b', Operator),
+            (r'[A-Z]' + _iwc + r'*|•' + _iwc + r'+', Operator),
             #
             # Constant
             # ========
@@ -102,8 +107,6 @@ class BQNLexer(RegexLexer):
             # ================
             (r'[;:?𝕨𝕩𝕗𝕘𝕤]', Name.Entity),
             #
-            
+
         ],
     }
-
-    
diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index 1bac3f5b..cf8a5fe8 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -121,11 +121,15 @@ class CSharpLexer(RegexLexer):
                     'while', 'get', 'set', 'new', 'partial', 'yield', 'add',
                     'remove', 'value', 'alias', 'ascending', 'descending',
                     'from', 'group', 'into', 'orderby', 'select', 'thenby',
-                    'where', 'join', 'equals', 'file', 'record', 'allows',
+                    'where', 'join', 'equals', 'record', 'allows',
                     'and', 'init', 'managed', 'nameof', 'nint', 'not',
                     'notnull', 'nuint', 'or', 'scoped', 'unmanaged', 'when',
                     'with'
                 ), suffix=r'\b'), Keyword),
+                # version 1: assumes that 'file' is the only contextual keyword
+                # that is a class modifier
+                (r'(file)(\s+)(record|class|abstract|enum|new|sealed|static)\b',
+                   bygroups(Keyword, Whitespace, Keyword)),
                 (r'(global)(::)', bygroups(Keyword, Punctuation)),
                 (r'(bool|byte|char|decimal|double|dynamic|float|int|long|object|'
                  r'sbyte|short|string|uint|ulong|ushort|var)\b\??', Keyword.Type),
diff --git a/pygments/lexers/igor.py b/pygments/lexers/igor.py
index 84ec32fd..6c9db672 100644
--- a/pygments/lexers/igor.py
+++ b/pygments/lexers/igor.py
@@ -38,7 +38,7 @@ class IgorLexer(RegexLexer):
     types = (
         'variable', 'string', 'constant', 'strconstant', 'NVAR', 'SVAR', 'WAVE',
         'STRUCT', 'dfref', 'funcref', 'char', 'uchar', 'int16', 'uint16', 'int32',
-        'uint32', 'int64', 'uint64', 'float', 'double'
+        'uint32', 'int64', 'uint64', 'float', 'double', 'int'
     )
     keywords = (
         'override', 'ThreadSafe', 'MultiThread', 'static',  'Proc',
diff --git a/pygments/lexers/macaulay2.py b/pygments/lexers/macaulay2.py
index 2120c876..f0ff57fd 100644
--- a/pygments/lexers/macaulay2.py
+++ b/pygments/lexers/macaulay2.py
@@ -13,9 +13,11 @@ from pygments.token import Comment, Keyword, Name, String, Text
 
 __all__ = ['Macaulay2Lexer']
 
-# Auto-generated for Macaulay2-1.24.05. Do not modify this file manually.
+# Auto-generated for Macaulay2-1.24.11. Do not modify this file manually.
 
 M2KEYWORDS = (
+    "SPACE",
+    "TEST",
     "and",
     "break",
     "catch",
@@ -37,7 +39,6 @@ M2KEYWORDS = (
     "or",
     "return",
     "shield",
-    "SPACE",
     "step",
     "symbol",
     "then",
@@ -54,52 +55,52 @@ M2KEYWORDS = (
     )
 
 M2DATATYPES = (
+    "ANCHOR",
     "Adjacent",
     "AffineVariety",
     "Analyzer",
-    "ANCHOR",
     "AngleBarList",
     "Array",
     "AssociativeExpression",
     "AtomicInt",
-    "Bag",
-    "BasicList",
-    "BettiTally",
-    "BinaryOperation",
     "BLOCKQUOTE",
     "BODY",
     "BOLD",
-    "Boolean",
     "BR",
     "BUTTON",
-    "CacheFunction",
-    "CacheTable",
+    "Bag",
+    "BasicList",
+    "BettiTally",
+    "BinaryOperation",
+    "Boolean",
     "CC",
     "CDATA",
+    "CODE",
+    "COMMENT",
+    "CacheFunction",
+    "CacheTable",
     "ChainComplex",
     "ChainComplexMap",
-    "CODE",
     "CoherentSheaf",
     "Command",
-    "COMMENT",
     "CompiledFunction",
     "CompiledFunctionBody",
     "CompiledFunctionClosure",
     "ComplexField",
     "Constant",
-    "Database",
     "DD",
+    "DIV",
+    "DL",
+    "DT",
+    "Database",
     "Descent",
     "Describe",
     "Dictionary",
     "DirectSum",
-    "DIV",
     "Divide",
-    "DL",
     "DocumentTag",
-    "DT",
-    "Eliminate",
     "EM",
+    "Eliminate",
     "EngineRing",
     "Equation",
     "ExampleItem",
@@ -118,7 +119,6 @@ M2DATATYPES = (
     "GradedModuleMap",
     "GroebnerBasis",
     "GroebnerBasisOptions",
-    "HashTable",
     "HEAD",
     "HEADER1",
     "HEADER2",
@@ -126,20 +126,23 @@ M2DATATYPES = (
     "HEADER4",
     "HEADER5",
     "HEADER6",
-    "HeaderType",
-    "Holder",
     "HR",
     "HREF",
     "HTML",
+    "HashTable",
+    "HeaderType",
+    "Holder",
     "Hybrid",
     "Hypertext",
     "HypertextContainer",
     "HypertextParagraph",
     "HypertextVoid",
-    "Ideal",
     "IMG",
-    "ImmutableType",
     "INDENT",
+    "INPUT",
+    "ITALIC",
+    "Ideal",
+    "ImmutableType",
     "IndeterminateNumber",
     "IndexedVariable",
     "IndexedVariableTable",
@@ -147,9 +150,7 @@ M2DATATYPES = (
     "InexactFieldFamily",
     "InexactNumber",
     "InfiniteNumber",
-    "INPUT",
     "IntermediateMarkUpType",
-    "ITALIC",
     "Iterator",
     "KBD",
     "Keyword",
@@ -157,17 +158,17 @@ M2DATATYPES = (
     "LATER",
     "LI",
     "LINK",
-    "List",
     "LITERAL",
+    "List",
     "LocalDictionary",
     "LowerBound",
+    "MENU",
+    "META",
     "Manipulator",
     "MapExpression",
     "MarkUpType",
     "Matrix",
     "MatrixExpression",
-    "MENU",
-    "META",
     "MethodFunction",
     "MethodFunctionBinary",
     "MethodFunctionSingle",
@@ -191,14 +192,14 @@ M2DATATYPES = (
     "Option",
     "OptionTable",
     "OrderedMonoid",
-    "Package",
     "PARA",
+    "PRE",
+    "Package",
     "Parenthesize",
     "Parser",
     "Partition",
     "PolynomialRing",
     "Power",
-    "PRE",
     "Product",
     "ProductOrder",
     "Program",
@@ -206,8 +207,11 @@ M2DATATYPES = (
     "ProjectiveHilbertPolynomial",
     "ProjectiveVariety",
     "Pseudocode",
+    "PseudocodeClosure",
     "QQ",
     "QuotientRing",
+    "RR",
+    "RRi",
     "RealField",
     "Resolution",
     "Ring",
@@ -215,10 +219,15 @@ M2DATATYPES = (
     "RingFamily",
     "RingMap",
     "RowExpression",
-    "RR",
-    "RRi",
     "SAMP",
     "SCRIPT",
+    "SMALL",
+    "SPAN",
+    "STRONG",
+    "STYLE",
+    "SUB",
+    "SUBSECTION",
+    "SUP",
     "ScriptedFunctor",
     "SelfInitializingType",
     "Sequence",
@@ -226,39 +235,32 @@ M2DATATYPES = (
     "SheafExpression",
     "SheafMap",
     "SheafOfRings",
-    "SMALL",
-    "SPAN",
     "SparseMonomialVectorExpression",
     "SparseVectorExpression",
     "String",
-    "STRONG",
-    "STYLE",
-    "SUB",
     "Subscript",
-    "SUBSECTION",
     "Sum",
     "SumOfTwists",
-    "SUP",
     "Superscript",
     "Symbol",
     "SymbolBody",
     "TABLE",
-    "Table",
-    "Tally",
-    "Task",
     "TD",
-    "TensorProduct",
-    "TestInput",
     "TEX",
     "TH",
-    "Thing",
-    "Time",
     "TITLE",
     "TO",
     "TO2",
     "TOH",
     "TR",
     "TT",
+    "Table",
+    "Tally",
+    "Task",
+    "TensorProduct",
+    "TestInput",
+    "Thing",
+    "Time",
     "Type",
     "UL",
     "URL",
@@ -270,11 +272,35 @@ M2DATATYPES = (
     "VirtualTally",
     "VisibleList",
     "WrapperType",
-    "ZeroExpression",
-    "ZZ"
+    "ZZ",
+    "ZeroExpression"
     )
 
 M2FUNCTIONS = (
+    "BesselJ",
+    "BesselY",
+    "Beta",
+    "Digamma",
+    "EXAMPLE",
+    "End",
+    "Fano",
+    "GCstats",
+    "GF",
+    "Gamma",
+    "Grassmannian",
+    "Hom",
+    "LLL",
+    "LUdecomposition",
+    "M2CODE",
+    "NNParser",
+    "Proj",
+    "QQParser",
+    "QRDecomposition",
+    "SVD",
+    "SYNOPSIS",
+    "Schubert",
+    "Spec",
+    "ZZParser",
     "about",
     "abs",
     "accumulate",
@@ -286,7 +312,6 @@ M2FUNCTIONS = (
     "addDependencyTask",
     "addEndFunction",
     "addHook",
-    "addStartFunction",
     "addStartTask",
     "adjoint",
     "agm",
@@ -317,10 +342,10 @@ M2FUNCTIONS = (
     "assert",
     "associatedGradedRing",
     "associatedPrimes",
+    "atEndOfFile",
     "atan",
     "atan2",
     "atanh",
-    "atEndOfFile",
     "autoload",
     "baseFilename",
     "baseName",
@@ -328,9 +353,6 @@ M2FUNCTIONS = (
     "basis",
     "beginDocumentation",
     "benchmark",
-    "BesselJ",
-    "BesselY",
-    "Beta",
     "betti",
     "between",
     "binomial",
@@ -343,9 +365,10 @@ M2FUNCTIONS = (
     "centerString",
     "chainComplex",
     "changeBase",
+    "changeDirectory",
     "char",
-    "characters",
     "charAnalyzer",
+    "characters",
     "check",
     "checkDegrees",
     "chi",
@@ -363,15 +386,15 @@ M2FUNCTIONS = (
     "cokernel",
     "collectGarbage",
     "columnAdd",
-    "columnate",
     "columnMult",
     "columnPermute",
     "columnRankProfile",
     "columnSwap",
+    "columnate",
     "combine",
     "commandInterpreter",
-    "commonest",
     "commonRing",
+    "commonest",
     "comodule",
     "compareExchange",
     "complement",
@@ -432,7 +455,6 @@ M2FUNCTIONS = (
     "dictionary",
     "diff",
     "difference",
-    "Digamma",
     "dim",
     "directSum",
     "disassemble",
@@ -452,7 +474,6 @@ M2FUNCTIONS = (
     "eint",
     "elements",
     "eliminate",
-    "End",
     "endPackage",
     "entries",
     "erase",
@@ -462,7 +483,6 @@ M2FUNCTIONS = (
     "euler",
     "eulers",
     "even",
-    "EXAMPLE",
     "examples",
     "exchange",
     "exec",
@@ -477,7 +497,6 @@ M2FUNCTIONS = (
     "extend",
     "exteriorPower",
     "factor",
-    "Fano",
     "fileExecutable",
     "fileExists",
     "fileLength",
@@ -510,14 +529,12 @@ M2FUNCTIONS = (
     "fromDual",
     "functionBody",
     "futureParser",
-    "Gamma",
     "gb",
     "gbRemove",
     "gbSnapshot",
     "gcd",
     "gcdCoefficients",
     "gcdLLL",
-    "GCstats",
     "genera",
     "generateAssertions",
     "generator",
@@ -528,9 +545,7 @@ M2FUNCTIONS = (
     "gens",
     "genus",
     "get",
-    "getc",
     "getChangeMatrix",
-    "getenv",
     "getGlobalSymbol",
     "getIOThreadMode",
     "getNetFile",
@@ -538,7 +553,8 @@ M2FUNCTIONS = (
     "getPrimeWithRootOfUnity",
     "getSymbol",
     "getWWW",
-    "GF",
+    "getc",
+    "getenv",
     "globalAssign",
     "globalAssignFunction",
     "globalAssignment",
@@ -548,11 +564,11 @@ M2FUNCTIONS = (
     "gramm",
     "graphIdeal",
     "graphRing",
-    "Grassmannian",
     "groebnerBasis",
     "groupID",
     "hash",
     "hashTable",
+    "headlines",
     "heft",
     "height",
     "hermite",
@@ -560,7 +576,6 @@ M2FUNCTIONS = (
     "hilbertPolynomial",
     "hilbertSeries",
     "hold",
-    "Hom",
     "homogenize",
     "homology",
     "homomorphism",
@@ -588,11 +603,11 @@ M2FUNCTIONS = (
     "input",
     "insert",
     "installAssignmentMethod",
-    "installedPackages",
     "installHilbertFunction",
     "installMethod",
     "installMinprimes",
     "installPackage",
+    "installedPackages",
     "instance",
     "instances",
     "integralClosure",
@@ -609,15 +624,14 @@ M2FUNCTIONS = (
     "inverseSystem",
     "irreducibleCharacteristicSeries",
     "irreducibleDecomposition",
-    "isAffineRing",
     "isANumber",
+    "isAffineRing",
     "isBorel",
-    "isc",
     "isCanceled",
     "isCommutative",
     "isConstant",
-    "isDirectory",
     "isDirectSum",
+    "isDirectory",
     "isEmpty",
     "isField",
     "isFinite",
@@ -631,10 +645,10 @@ M2FUNCTIONS = (
     "isInputFile",
     "isIsomorphic",
     "isIsomorphism",
+    "isLLL",
     "isLiftable",
     "isLinearType",
     "isListener",
-    "isLLL",
     "isMember",
     "isModule",
     "isMonomialIdeal",
@@ -657,6 +671,7 @@ M2FUNCTIONS = (
     "isRegularFile",
     "isRing",
     "isSkewCommutative",
+    "isSmooth",
     "isSorted",
     "isSquareFree",
     "isStandardGradedPolynomialRing",
@@ -667,8 +682,10 @@ M2FUNCTIONS = (
     "isSurjective",
     "isTable",
     "isUnit",
+    "isVeryAmple",
     "isWellDefined",
     "isWeylAlgebra",
+    "isc",
     "iterator",
     "jacobian",
     "jacobianDual",
@@ -697,7 +714,6 @@ M2FUNCTIONS = (
     "linkFile",
     "listForm",
     "listSymbols",
-    "LLL",
     "lngamma",
     "load",
     "loadPackage",
@@ -708,8 +724,6 @@ M2FUNCTIONS = (
     "log1p",
     "lookup",
     "lookupCount",
-    "LUdecomposition",
-    "M2CODE",
     "makeDirectory",
     "makeDocumentTag",
     "makePackageIndex",
@@ -732,6 +746,8 @@ M2FUNCTIONS = (
     "methods",
     "midpoint",
     "min",
+    "minPosition",
+    "minPres",
     "mingens",
     "mingle",
     "minimalBetti",
@@ -741,8 +757,6 @@ M2FUNCTIONS = (
     "minimize",
     "minimizeFilename",
     "minors",
-    "minPosition",
-    "minPres",
     "minprimes",
     "minus",
     "mkdir",
@@ -752,8 +766,8 @@ M2FUNCTIONS = (
     "monoid",
     "monomialCurveIdeal",
     "monomialIdeal",
-    "monomials",
     "monomialSubideal",
+    "monomials",
     "moveFile",
     "multidegree",
     "multidoc",
@@ -773,24 +787,23 @@ M2FUNCTIONS = (
     "newPackage",
     "newRing",
     "next",
-    "nextkey",
     "nextPrime",
-    "NNParser",
+    "nextkey",
     "nonspaceAnalyzer",
     "norm",
     "normalCone",
     "notImplemented",
-    "nullhomotopy",
     "nullParser",
     "nullSpace",
+    "nullhomotopy",
+    "numColumns",
+    "numRows",
     "number",
     "numcols",
-    "numColumns",
     "numerator",
     "numeric",
     "numericInterval",
     "numgens",
-    "numRows",
     "numrows",
     "odd",
     "oeis",
@@ -804,9 +817,9 @@ M2FUNCTIONS = (
     "openListener",
     "openOut",
     "openOutAppend",
+    "optP",
     "optionalSignParser",
     "options",
-    "optP",
     "orP",
     "override",
     "pack",
@@ -844,23 +857,20 @@ M2FUNCTIONS = (
     "primaryComponent",
     "primaryDecomposition",
     "print",
-    "printerr",
     "printString",
+    "printerr",
     "processID",
     "product",
     "profile",
-    "Proj",
     "projectiveHilbertPolynomial",
     "promote",
     "protect",
     "prune",
-    "pseudocode",
     "pseudoRemainder",
+    "pseudocode",
     "pullback",
     "pushForward",
     "pushout",
-    "QQParser",
-    "QRDecomposition",
     "quotient",
     "quotientRemainder",
     "radical",
@@ -872,21 +882,21 @@ M2FUNCTIONS = (
     "rays",
     "read",
     "readDirectory",
-    "readlink",
     "readPackage",
+    "readlink",
     "realPart",
     "realpath",
     "recursionDepth",
-    "reducedRowEchelonForm",
     "reduceHilbert",
+    "reducedRowEchelonForm",
     "reductionNumber",
     "reesAlgebra",
     "reesAlgebraIdeal",
     "reesIdeal",
+    "regSeqInIdeal",
     "regex",
     "regexQuote",
     "registerFinalizer",
-    "regSeqInIdeal",
     "regularity",
     "regularizedBeta",
     "regularizedGamma",
@@ -929,13 +939,15 @@ M2FUNCTIONS = (
     "scanValues",
     "schedule",
     "schreyerOrder",
-    "Schubert",
     "searchPath",
     "sec",
     "sech",
     "seeParsing",
     "select",
     "selectInSubring",
+    "selectKeys",
+    "selectPairs",
+    "selectValues",
     "selectVariables",
     "separate",
     "separateRegexp",
@@ -950,6 +962,8 @@ M2FUNCTIONS = (
     "setRandomSeed",
     "setup",
     "setupEmacs",
+    "setupLift",
+    "setupPromote",
     "sheaf",
     "sheafHom",
     "sheafMap",
@@ -970,7 +984,6 @@ M2FUNCTIONS = (
     "sortColumns",
     "source",
     "span",
-    "Spec",
     "specialFiber",
     "specialFiberIdeal",
     "splice",
@@ -996,7 +1009,6 @@ M2FUNCTIONS = (
     "sum",
     "super",
     "support",
-    "SVD",
     "switch",
     "sylvesterMatrix",
     "symbolBody",
@@ -1007,7 +1019,6 @@ M2FUNCTIONS = (
     "symmetricKernel",
     "symmetricPower",
     "synonym",
-    "SYNOPSIS",
     "syz",
     "syzygyScheme",
     "table",
@@ -1024,7 +1035,6 @@ M2FUNCTIONS = (
     "tensorAssociativity",
     "terminalParser",
     "terms",
-    "TEST",
     "testHunekeQuestion",
     "tests",
     "tex",
@@ -1038,14 +1048,14 @@ M2FUNCTIONS = (
     "toField",
     "toList",
     "toLower",
-    "top",
-    "topCoefficients",
-    "topComponents",
     "toRR",
     "toRRi",
     "toSequence",
     "toString",
     "toUpper",
+    "top",
+    "topCoefficients",
+    "topComponents",
     "trace",
     "transpose",
     "trim",
@@ -1086,12 +1096,12 @@ M2FUNCTIONS = (
     "wrap",
     "youngest",
     "zero",
-    "zeta",
-    "ZZParser"
+    "zeta"
     )
 
 M2CONSTANTS = (
     "A1BrouwerDegrees",
+    "AbstractSimplicialComplexes",
     "AbstractToricVarieties",
     "Acknowledgement",
     "AdditionalPaths",
@@ -1157,6 +1167,7 @@ M2CONSTANTS = (
     "CharacteristicClasses",
     "CheckDocumentation",
     "Chordal",
+    "cite",
     "Classic",
     "clearAll",
     "clearOutput",
@@ -1401,6 +1412,7 @@ M2CONSTANTS = (
     "LongPolynomial",
     "M0nbar",
     "Macaulay2Doc",
+    "Maintainer",
     "MakeDocumentation",
     "MakeHTML",
     "MakeInfo",
@@ -1436,7 +1448,9 @@ M2CONSTANTS = (
     "MonomialOrder",
     "Monomials",
     "MonomialSize",
+    "Msolve",
     "MultigradedBGG",
+    "MultigradedImplicitization",
     "MultiGradedRationalMap",
     "MultiplicitySequence",
     "MultiplierIdeals",
@@ -1470,6 +1484,7 @@ M2CONSTANTS = (
     "NumericalImplicitization",
     "NumericalLinearAlgebra",
     "NumericalSchubertCalculus",
+    "NumericalSemigroups",
     "NumericSolutions",
     "numTBBThreads",
     "OIGroebnerBases",
@@ -1502,6 +1517,7 @@ M2CONSTANTS = (
     "path",
     "PencilsOfQuadrics",
     "Permanents",
+    "Permutations",
     "PHCpack",
     "PhylogeneticTrees",
     "pi",
@@ -1612,6 +1628,7 @@ M2CONSTANTS = (
     "SchurFunctors",
     "SchurRings",
     "SchurVeronese",
+    "SCMAlgebras",
     "scriptCommandLine",
     "SCSCP",
     "SectionRing",
@@ -1688,6 +1705,7 @@ M2CONSTANTS = (
     "Text",
     "ThinSincereQuivers",
     "ThreadedGB",
+    "Threads",
     "Threshold",
     "Topcom",
     "topLevelMode",
diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py
index 65082a07..48fa580d 100644
--- a/pygments/lexers/matlab.py
+++ b/pygments/lexers/matlab.py
@@ -31,7 +31,7 @@ class MatlabLexer(RegexLexer):
     url = 'https://www.mathworks.com/products/matlab.html'
     version_added = '0.10'
 
-    _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\.\\|\./|/|\\'
+    _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\^|\.\\|\./|/|\\'
 
     tokens = {
         'expressions': [
@@ -3175,7 +3175,7 @@ class OctaveLexer(RegexLexer):
             # operators in Octave but not Matlab requiring escape for re:
             (r'\*=|\+=|\^=|\/=|\\=|\*\*|\+\+|\.\*\*', Operator),
             # operators requiring escape for re:
-            (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator),
+            (r'\.\*|\*|\+|\.\^|\^|\.\\|\.\/|\/|\\', Operator),
 
 
             # punctuation:
@@ -3260,7 +3260,7 @@ class ScilabLexer(RegexLexer):
             # operators:
             (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
             # operators requiring escape for re:
-            (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator),
+            (r'\.\*|\*|\+|\.\^|\^|\.\\|\.\/|\/|\\', Operator),
 
             # punctuation:
             (r'[\[\](){}@.,=:;]+', Punctuation),
diff --git a/pygments/lexers/pddl.py b/pygments/lexers/pddl.py
new file mode 100644
index 00000000..ae978444
--- /dev/null
+++ b/pygments/lexers/pddl.py
@@ -0,0 +1,82 @@
+"""
+    pygments.lexers.pddl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Planning Domain Definition Language.
+
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Punctuation, Keyword, Whitespace, Name, Comment, \
+    Operator, Number
+
+
+__all__ = ['PddlLexer']
+
+
+class PddlLexer(RegexLexer):
+    """
+    A PDDL lexer.
+
+    It should support up to PDDL 3.1.
+    """
+
+    name = 'PDDL'
+    aliases = ['pddl']
+    filenames = ['*.pddl']
+    # there doesn't really seem to be a PDDL homepage.
+    url = 'https://en.wikipedia.org/wiki/Planning_Domain_Definition_Language'
+    version_added = '2.19'
+
+    tokens = {
+        'root': [
+            (r'\s+', Whitespace),
+            (r';.*$', Comment.Singleline),
+            include('keywords'),
+            include('builtins'),
+            (r'[()]', Punctuation),
+            (r'[=/*+><-]', Operator),
+            (r'[a-zA-Z][a-zA-Z0-9_-]*', Name),
+            (r'\?[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable),
+            (r'[0-9]+\.[0-9]+', Number.Float),
+            (r'[0-9]+', Number.Integer),
+        ],
+        'keywords': [
+            (words((
+                ':requirements', ':types', ':constants',
+                ':predicates', ':functions', ':action', ':agent',
+                ':parameters', ':precondition', ':effect',
+                ':durative-action', ':duration', ':condition',
+                ':derived', ':domain', ':objects', ':init',
+                ':goal', ':metric', ':length', ':serial', ':parallel',
+                # the following are requirements
+                ':strips', ':typing', ':negative-preconditions',
+                ':disjunctive-preconditions', ':equality',
+                ':existential-preconditions', ':universal-preconditions',
+                ':conditional-effects', ':fluents', ':numeric-fluents',
+                ':object-fluents', ':adl', ':durative-actions',
+                ':continuous-effects', ':derived-predicates',
+                ':time-intial-literals', ':preferences',
+                ':constraints', ':action-costs', ':multi-agent',
+                ':unfactored-privacy', ':factored-privacy',
+                ':non-deterministic'
+                ), suffix=r'\b'), Keyword)
+        ],
+        'builtins': [
+            (words((
+                'define', 'domain', 'object', 'either', 'and',
+                'forall', 'preference', 'imply', 'or', 'exists',
+                'not', 'when', 'assign', 'scale-up', 'scale-down',
+                'increase', 'decrease', 'at', 'over', 'start',
+                'end', 'all', 'problem', 'always', 'sometime',
+                'within', 'at-most-once', 'sometime-after',
+                'sometime-before', 'always-within', 'hold-during',
+                'hold-after', 'minimize', 'maximize',
+                'total-time', 'is-violated'), suffix=r'\b'),
+                Name.Builtin)
+        ]
+    }
+
diff --git a/pygments/lexers/rego.py b/pygments/lexers/rego.py
new file mode 100644
index 00000000..0cc6ab3b
--- /dev/null
+++ b/pygments/lexers/rego.py
@@ -0,0 +1,58 @@
+"""
+    pygments.lexers.rego
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Rego policy languages.
+
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, Number, Punctuation, Whitespace
+
+class RegoLexer(RegexLexer):
+    """
+    For Rego source.
+    """
+    name = 'Rego'
+    url = 'https://www.openpolicyagent.org/docs/latest/policy-language/'
+    filenames = ['*.rego']
+    aliases = ['rego']
+    mimetypes = ['text/x-rego']
+    version_added = '2.18'
+
+    reserved_words = (
+        'as', 'contains', 'data', 'default', 'else', 'every', 'false',
+        'if', 'in', 'import', 'package', 'not', 'null',
+        'some', 'true', 'with'
+    )
+
+    builtins = (
+        # https://www.openpolicyagent.org/docs/latest/philosophy/#the-opa-document-model
+        'data',  # Global variable for accessing base and virtual documents
+        'input', # Represents synchronously pushed base documents
+    )
+
+    tokens = {
+        'root': [
+            (r'\n', Text.Whitespace),
+            (r'\s+', Text),
+            (r'#.*?$', Comment.Single),
+            (words(reserved_words, suffix=r'\b'), Keyword),
+            (words(builtins, suffix=r'\b'), Name.Builtin),
+            (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r'`[^`]*`', String.Backtick),
+            (r'-?\d+(\.\d+)?', Number),
+            (r'(==|!=|<=|>=|:=)', Operator),  # Compound operators
+            (r'[=<>+\-*/%&|]', Operator),     # Single-character operators
+            (r'[\[\]{}(),.:;]', Punctuation),
+        ]
+    }
+
+__all__ = ['RegoLexer']
+
+
+
diff --git a/pygments/lexers/ruby.py b/pygments/lexers/ruby.py
index 134ff779..b8a928bb 100644
--- a/pygments/lexers/ruby.py
+++ b/pygments/lexers/ruby.py
@@ -253,7 +253,8 @@ class RubyLexer(ExtendedRegexLexer):
              heredoc_callback),
             # empty string heredocs
             (r'(<<[-~]?)("|\')()(\2)(.*?\n)', heredoc_callback),
-            (r'__END__', Comment.Preproc, 'end-part'),
+            # ensure << without space is not treated as heredoc
+            (r'(?<!\w)(<<)(?=[^-\w])', Operator),
             # multiline regex (after keywords or assignments)
             (r'(?:^|(?<=[=<>~!:])|'
              r'(?<=(?:\s|;)when\s)|'
diff --git a/pygments/lexers/scripting.py b/pygments/lexers/scripting.py
index d38387e0..8b7c4e55 100644
--- a/pygments/lexers/scripting.py
+++ b/pygments/lexers/scripting.py
@@ -21,6 +21,10 @@ __all__ = ['LuaLexer', 'LuauLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLe
            'EasytrieveLexer', 'JclLexer', 'MiniScriptLexer']
 
 
+def all_lua_builtins():
+    from pygments.lexers._lua_builtins import MODULES
+    return [w for values in MODULES.values() for w in values]
+
 class LuaLexer(RegexLexer):
     """
     For Lua source code.
@@ -66,7 +70,7 @@ class LuaLexer(RegexLexer):
         'ws': [
             (_comment_multiline, Comment.Multiline),
             (_comment_single, Comment.Single),
-            (_space, Text),
+            (_space, Whitespace),
         ],
         'base': [
             include('ws'),
@@ -93,12 +97,23 @@ class LuaLexer(RegexLexer):
 
             (r'(function)\b', Keyword.Reserved, 'funcname'),
 
-            (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name),
+            (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
+            (fr'[A-Za-z_]\w*(?={_s}*[.:])', Name.Variable, 'varname'),
+            (fr'[A-Za-z_]\w*(?={_s}*\()', Name.Function),
+            (r'[A-Za-z_]\w*', Name.Variable),
 
             ("'", String.Single, combined('stringescape', 'sqs')),
             ('"', String.Double, combined('stringescape', 'dqs'))
         ],
 
+        'varname': [
+            include('ws'),
+            (r'[.:]', Punctuation),
+            (rf'{_name}(?={_s}*[.:])', Name.Property),
+            (rf'{_name}(?={_s}*\()', Name.Function, '#pop'),
+            (_name, Name.Property, '#pop'),
+        ],
+
         'funcname': [
             include('ws'),
             (r'[.:]', Punctuation),
@@ -151,16 +166,15 @@ class LuaLexer(RegexLexer):
     def get_tokens_unprocessed(self, text):
         for index, token, value in \
                 RegexLexer.get_tokens_unprocessed(self, text):
-            if token is Name:
-                if value in self._functions:
-                    yield index, Name.Builtin, value
-                    continue
-                elif '.' in value:
+            if token is Name.Builtin and value not in self._functions:
+                if '.' in value:
                     a, b = value.split('.')
                     yield index, Name, a
                     yield index + len(a), Punctuation, '.'
                     yield index + len(a) + 1, Name, b
-                    continue
+                else:
+                    yield index, Name, value
+                continue
             yield index, token, value
 
 def _luau_make_expression(should_pop, _s):
@@ -521,7 +535,8 @@ class MoonScriptLexer(LuaLexer):
             (r'(self)\b', Name.Builtin.Pseudo),
             (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class),
             (r'[A-Z]\w*', Name.Class),  # proper name
-            (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name),
+            (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
+            (r'[A-Za-z_]\w*', Name),
             ("'", String.Single, combined('stringescape', 'sqs')),
             ('"', String.Double, combined('stringescape', 'dqs'))
         ],
diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py
index 90165389..56a926d3 100644
--- a/pygments/lexers/sql.py
+++ b/pygments/lexers/sql.py
@@ -40,6 +40,7 @@
     :license: BSD, see LICENSE for details.
 """
 
+import collections
 import re
 
 from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words
@@ -56,12 +57,13 @@ from pygments.lexers._mysql_builtins import \
     MYSQL_KEYWORDS, \
     MYSQL_OPTIMIZER_HINTS
 
+from pygments.lexers import _googlesql_builtins
 from pygments.lexers import _tsql_builtins
 
 
-__all__ = ['PostgresLexer', 'PlPgsqlLexer', 'PostgresConsoleLexer',
-           'PostgresExplainLexer', 'SqlLexer', 'TransactSqlLexer',
-           'MySqlLexer', 'SqliteConsoleLexer', 'RqlLexer']
+__all__ = ['GoogleSqlLexer', 'PostgresLexer', 'PlPgsqlLexer',
+           'PostgresConsoleLexer', 'PostgresExplainLexer', 'SqlLexer',
+           'TransactSqlLexer', 'MySqlLexer', 'SqliteConsoleLexer', 'RqlLexer']
 
 line_re  = re.compile('.*?\n')
 sqlite_prompt_re = re.compile(r'^(?:sqlite|   ...)>(?= )')
@@ -77,6 +79,12 @@ tsql_go_re = re.compile(r'\bgo\b', re.IGNORECASE)
 tsql_declare_re = re.compile(r'\bdeclare\s+@', re.IGNORECASE)
 tsql_variable_re = re.compile(r'@[a-zA-Z_]\w*\b')
 
+# Identifiers for analyse_text()
+googlesql_identifiers = (
+    _googlesql_builtins.functionnames
+    + _googlesql_builtins.keywords
+    + _googlesql_builtins.types)
+
 
 def language_callback(lexer, match):
     """Parse the content of a $-string using a lexer
@@ -964,6 +972,166 @@ class MySqlLexer(RegexLexer):
         return rating
 
 
+class GoogleSqlLexer(RegexLexer):
+    """
+    GoogleSQL is Google's standard SQL dialect, formerly known as ZetaSQL.
+
+    The list of keywords includes reserved words for future use.
+    """
+
+    name = 'GoogleSQL'
+    aliases = ['googlesql', 'zetasql']
+    filenames = ['*.googlesql', '*.googlesql.sql']
+    mimetypes = ['text/x-google-sql', 'text/x-google-sql-aux']
+    url = 'https://cloud.google.com/bigquery/googlesql'
+    version_added = '2.19'
+
+    flags = re.IGNORECASE
+    tokens = {
+        'root': [
+            (r'\s+', Whitespace),
+
+            # Comments
+            (r'(?:#|--\s+).*', Comment.Single),
+            (r'/\*', Comment.Multiline, 'multiline-comment'),
+
+            # Hexadecimal literals
+            (r"x'([0-9a-f]{2})+'", Number.Hex),
+            (r'0x[0-9a-f]+', Number.Hex),
+
+            # Binary literals
+            (r"b'[01]+'", Number.Bin),
+            (r'0b[01]+', Number.Bin),
+
+            # Numeric literals
+            (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float),  # Mandatory integer, optional fraction and exponent
+            (r'[0-9]*\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),  # Mandatory fraction, optional integer and exponent
+            (r'[0-9]+e[+-]?[0-9]+', Number.Float),  # Exponents with integer significands are still floats
+            (r'[0-9]+(?=[^0-9a-z$_\u0080-\uffff])', Number.Integer),  # Integers that are not in a schema object name
+
+            # Date literals
+            (r"\{\s*d\s*(?P<quote>['\"])\s*\d{2}(\d{2})?.?\d{2}.?\d{2}\s*(?P=quote)\s*\}",
+             Literal.Date),
+
+            # Time literals
+            (r"\{\s*t\s*(?P<quote>['\"])\s*(?:\d+\s+)?\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?\s*(?P=quote)\s*\}",
+             Literal.Date),
+
+            # Timestamp literals
+            (
+                r"\{\s*ts\s*(?P<quote>['\"])\s*"
+                r"\d{2}(?:\d{2})?.?\d{2}.?\d{2}"  # Date part
+                r"\s+"  # Whitespace between date and time
+                r"\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?"  # Time part
+                r"\s*(?P=quote)\s*\}",
+                Literal.Date
+            ),
+
+            # String literals
+            (r"'", String.Single, 'single-quoted-string'),
+            (r'"', String.Double, 'double-quoted-string'),
+
+            # Variables
+            (r'@@(?:global\.|persist\.|persist_only\.|session\.)?[a-z_]+', Name.Variable),
+            (r'@[a-z0-9_$.]+', Name.Variable),
+            (r"@'", Name.Variable, 'single-quoted-variable'),
+            (r'@"', Name.Variable, 'double-quoted-variable'),
+            (r"@`", Name.Variable, 'backtick-quoted-variable'),
+            (r'\?', Name.Variable),  # For demonstrating prepared statements
+
+            # Exceptions; these words tokenize differently in different contexts.
+            (r'\b(set)(?!\s*\()', Keyword),
+            (r'\b(character)(\s+)(set)\b', bygroups(Keyword, Whitespace, Keyword)),
+
+            # Constants, types, keywords, functions, operators
+            (words(_googlesql_builtins.constants, prefix=r'\b', suffix=r'\b'), Name.Constant),
+            (words(_googlesql_builtins.types, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+            (words(_googlesql_builtins.keywords, prefix=r'\b', suffix=r'\b'), Keyword),
+            (words(_googlesql_builtins.functionnames, prefix=r'\b', suffix=r'\b(\s*)(\()'),
+             bygroups(Name.Function, Whitespace, Punctuation)),
+            (words(_googlesql_builtins.operators, prefix=r'\b', suffix=r'\b'), Operator),
+
+            # Schema object names
+            #
+            # Note: Although the first regex supports unquoted all-numeric
+            # identifiers, this will not be a problem in practice because
+            # numeric literals have already been handled above.
+            #
+            ('[0-9a-z$_\u0080-\uffff]+', Name),
+            (r'`', Name.Quoted, 'schema-object-name'),
+
+            # Punctuation
+            (r'[(),.;]', Punctuation),
+        ],
+
+        # Multiline comment substates
+        # ---------------------------
+
+        'multiline-comment': [
+            (r'[^*]+', Comment.Multiline),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+
+        # String substates
+        # ----------------
+
+        'single-quoted-string': [
+            (r"[^'\\]+", String.Single),
+            (r"''", String.Escape),
+            (r"""\\[0'"bnrtZ\\%_]""", String.Escape),
+            (r"'", String.Single, '#pop'),
+        ],
+
+        'double-quoted-string': [
+            (r'[^"\\]+', String.Double),
+            (r'""', String.Escape),
+            (r"""\\[0'"bnrtZ\\%_]""", String.Escape),
+            (r'"', String.Double, '#pop'),
+        ],
+
+        # Variable substates
+        # ------------------
+
+        'single-quoted-variable': [
+            (r"[^']+", Name.Variable),
+            (r"''", Name.Variable),
+            (r"'", Name.Variable, '#pop'),
+        ],
+
+        'double-quoted-variable': [
+            (r'[^"]+', Name.Variable),
+            (r'""', Name.Variable),
+            (r'"', Name.Variable, '#pop'),
+        ],
+
+        'backtick-quoted-variable': [
+            (r'[^`]+', Name.Variable),
+            (r'``', Name.Variable),
+            (r'`', Name.Variable, '#pop'),
+        ],
+
+        # Schema object name substates
+        # ----------------------------
+        #
+        # "Name.Quoted" and "Name.Quoted.Escape" are non-standard but
+        # formatters will style them as "Name" by default but add
+        # additional styles based on the token name. This gives users
+        # flexibility to add custom styles as desired.
+        #
+        'schema-object-name': [
+            (r'[^`]+', Name.Quoted),
+            (r'``', Name.Quoted.Escape),
+            (r'`', Name.Quoted, '#pop'),
+        ],
+    }
+
+    def analyse_text(text):
+        tokens = collections.Counter(text.split())
+        return 0.001 * sum(count for t, count in tokens.items()
+                           if t in googlesql_identifiers)
+
+
 class SqliteConsoleLexer(Lexer):
     """
     Lexer for example sessions using sqlite3.


Test patch:
diff --git a/tests/examplefiles/matlab/matlab_sample.m b/tests/examplefiles/matlab/matlab_sample.m
index 86163166..c2e8eeb6 100644
--- a/tests/examplefiles/matlab/matlab_sample.m
+++ b/tests/examplefiles/matlab/matlab_sample.m
@@ -14,7 +14,7 @@ a = rand(30);
 b = rand(30);
 
 c = a .* b ./ a \ ... comment at end of line and continuation
-    (b .* a + b - a);
+    (b .* a + b - a) ^ 2;
 
 c = a' * b';  % note: these ticks are for transpose, not quotes.
 
diff --git a/tests/snippets/bqn/test_inter_word.txt b/tests/snippets/bqn/test_inter_word.txt
new file mode 100644
index 00000000..82b72c34
--- /dev/null
+++ b/tests/snippets/bqn/test_inter_word.txt
@@ -0,0 +1,13 @@
+---input---
+{ab𝕊is: ab}
+
+---tokens---
+'{'           Keyword.Type
+'ab'          Name.Variable
+'𝕊'           Operator
+'is'          Name.Variable
+':'           Name.Entity
+' '           Text.Whitespace
+'ab'          Name.Variable
+'}'           Keyword.Type
+'\n'          Text.Whitespace
diff --git a/tests/snippets/bqn/test_number_underscores.txt b/tests/snippets/bqn/test_number_underscores.txt
new file mode 100644
index 00000000..b87f9b8b
--- /dev/null
+++ b/tests/snippets/bqn/test_number_underscores.txt
@@ -0,0 +1,12 @@
+---input---
+⟨1_.0__1_,10_000,_1⟩
+
+---tokens---
+'⟨'           Punctuation
+'1_.0__1_'    Literal.Number
+','           Punctuation
+'10_000'      Literal.Number
+','           Punctuation
+'_1'          Name.Attribute
+'⟩'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/bqn/test_recursive_modifier.txt b/tests/snippets/bqn/test_recursive_modifier.txt
new file mode 100644
index 00000000..05aa2af4
--- /dev/null
+++ b/tests/snippets/bqn/test_recursive_modifier.txt
@@ -0,0 +1,10 @@
+---input---
+_𝕣 𝕣 _𝕣_
+
+---tokens---
+'_𝕣'          Name.Attribute
+' '           Text.Whitespace
+'𝕣'           Name.Attribute
+' '           Text.Whitespace
+'_𝕣_'         Name.Property
+'\n'          Text.Whitespace
diff --git a/tests/snippets/csharp/test_file_keyword.txt b/tests/snippets/csharp/test_file_keyword.txt
new file mode 100644
index 00000000..44226cc0
--- /dev/null
+++ b/tests/snippets/csharp/test_file_keyword.txt
@@ -0,0 +1,141 @@
+---input---
+file class X { }
+file record X(int file);
+file abstract class X { }
+file sealed class X { }
+file static class X { }
+file enum X { file }
+file new record X(int file);
+int file(Func<int> file) => file();
+int file = Program.file(file => 42);
+
+---tokens---
+'file'        Keyword
+' '           Text.Whitespace
+'class'       Keyword
+' '           Text.Whitespace
+'X'           Name
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'file'        Keyword
+' '           Text.Whitespace
+'record'      Keyword
+' '           Text.Whitespace
+'X'           Name.Function
+'('           Punctuation
+'int'         Keyword.Type
+' '           Text.Whitespace
+'file'        Name
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'file'        Keyword
+' '           Text.Whitespace
+'abstract'    Keyword
+' '           Text.Whitespace
+'class'       Keyword
+' '           Text.Whitespace
+'X'           Name.Class
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'file'        Keyword
+' '           Text.Whitespace
+'sealed'      Keyword
+' '           Text.Whitespace
+'class'       Keyword
+' '           Text.Whitespace
+'X'           Name.Class
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'file'        Keyword
+' '           Text.Whitespace
+'static'      Keyword
+' '           Text.Whitespace
+'class'       Keyword
+' '           Text.Whitespace
+'X'           Name.Class
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'file'        Keyword
+' '           Text.Whitespace
+'enum'        Keyword
+' '           Text.Whitespace
+'X'           Name
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'file'        Name
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'file'        Keyword
+' '           Text.Whitespace
+'new'         Keyword
+' '           Text.Whitespace
+'record'      Keyword
+' '           Text.Whitespace
+'X'           Name.Function
+'('           Punctuation
+'int'         Keyword.Type
+' '           Text.Whitespace
+'file'        Name
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'int'         Keyword.Type
+' '           Text.Whitespace
+'file'        Name.Function
+'('           Punctuation
+'Func'        Name
+'<'           Operator
+'int'         Keyword.Type
+'>'           Operator
+' '           Text.Whitespace
+'file'        Name
+')'           Punctuation
+' '           Text.Whitespace
+'=>'          Operator
+' '           Text.Whitespace
+'file'        Name
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'int'         Keyword.Type
+' '           Text.Whitespace
+'file'        Name
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'Program'     Name
+'.'           Punctuation
+'file'        Name
+'('           Punctuation
+'file'        Name
+' '           Text.Whitespace
+'=>'          Operator
+' '           Text.Whitespace
+'42'          Literal.Number.Integer
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/googlesql/googlesql_test.txt b/tests/snippets/googlesql/googlesql_test.txt
new file mode 100644
index 00000000..b69a64c3
--- /dev/null
+++ b/tests/snippets/googlesql/googlesql_test.txt
@@ -0,0 +1,1485 @@
+---input---
+BEGIN TRANSACTION
+CREATE TEMP TABLE Flights(total INT64)  AS SELECT * FROM UNNEST([10,23,3,14,55]) AS a;
+SELECT *, SPLIT_SUBSTR('blah', 3||4) FROM Flights;
+
+## function calls and builtins
+
+SELECT abs(1.5);
+CREATE OR REPLACE my_table
+SELECT REPLACE('good show', 'good', 'great');
+SELECT LPAD("hello", 10);
+SELECT D3A_COUNT.TO_HLL();
+SELECT KLL_QUANTILES.MERGE_UINT64();
+SELECT MyNonBuiltinFunction(an_arg);
+SELECT MyNonBuiltinFunction((SELECT 1), an_arg), Baz();
+SELECT MyNonBuiltinFunction(')'), Baz(an_arg);
+SELECT (1)
+DEFINE TABLE Foo (options);
+DEFINE MACRO FOO the body of the macro with first arg $1 and second arg $2;
+SELECT $mymacro FROM my_table;
+SELECT "A macro inside a string: $mymacro" FROM my_table;
+SELECT 'A macro inside a string: $mymacro' FROM my_table;
+SELECT """A macro inside a string: $mymacro""" FROM my_table;
+SELECT rb"""A macro inside a string: $mymacro("arg1", "arg2")""" FROM my_table;
+select creation, latency from my_table_${YYYYMMDD+1};
+select creation, latency from my_table_${YYYY+2};
+select creation, latency from my_table_${YYYYMMDD-3};
+select creation, latency from my_table_${DD};
+select "This is the day of the month ${DD}. QED.";
+select * from logs limit ${LimitCount};
+SELECT 1 = 1, 1 != 42;
+SELECT 1 > 2;
+LOAD my.script;
+
+## Ambiguous keywords, either type or function or keywords.
+
+SELECT JSON '{"name": "Alice", "age": 30}';
+SELECT IF("%.1f", 1/3);
+CREATE TABLE IF NOT EXISTS my_table [(my_column int64)];
+SELECT EXTRACT(DAY FROM DATE '2013-12-25');
+SELECT CAST(1 AS BOOL);
+SELECT element from UNNEST([1, 2, 3] as element);
+select [1, 2, 3][OFFSET(0)];
+SELECT * FROM UNNEST(['a', 'b']) AS letter
+ORDER BY letter ASC LIMIT 1 OFFSET 1;
+SELECT * FROM Produce PIVOT(SUM(sales) FOR quarter IN ('Q1', 'Q2', 'Q3', 'Q4'))
+SELECT * FROM Produce UNPIVOT(sales FOR quarter IN (Q1, Q2, Q3, Q4))
+
+## comments
+
+SELECT TRUE;  -- double-dash comment
+SELECT TRUE;  # hash comment
+SELECT TRUE;  // this is a comment
+SELECT /* this is block comment */ TRUE;
+# Bracket comments in stock SQL VS Code config are not supported in GoogleSQL
+SELECT %{bracket comment};
+
+## embedding JS
+
+CREATE FUNCTION f(x INT, y INT)
+RETURNS DOUBLE LANGUAGE js AS """return (x*y);""";
+
+## embedding Lua
+
+CREATE FUNCTION f(x INT, y INT)
+RETURNS DOUBLE LANGUAGE lua AS """return (x*y);""";
+
+### strings
+
+SELECT 'Hi !';
+SELECT "Hi !";
+SELECT 'a tab \t in a single-quoted string';
+SELECT "a tab \t in a single-quoted string";
+SELECT "string with a # sign";
+SELECT 'string with a # sign';
+SELECT 'a string with a \' single quote';
+SELECT "a string with a \" double quote";
+SELECT """triple double quote""";
+SELECT '''triple single quote''';
+SELECT 'triple single quote no word boundary';
+SELECT "triple single quote no word boundary";
+SELECT '''triple single quote no word boundary''';
+SELECT b'binary single quoted string';
+SELECT B'binary single quoted string';
+SELECT b"binary double quoted string";
+SELECT B"binary double quoted string";
+SELECT b'''triple single quote''';
+SELECT B'''triple single quote''';
+SELECT b"""triple double quote""";
+SELECT B"""triple double quote""";
+SELECT r"raw single quoted string";
+SELECT r"double quote";
+SELECT rb"double quote";
+SELECT rB"double quote";
+SELECT R"double quote";
+SELECT Rb"double quote";
+SELECT RB"double quote";
+SELECT r'single quote';
+SELECT rb'single quote';
+SELECT rB'single quote';
+SELECT R'single quote';
+SELECT Rb'single quote';
+SELECT RB'single quote';
+SELECT r'single quote with an uninterpreted \' escape';
+SELECT b'single quote with an *interpreted* \' escape';
+SELECT r"""raw single quoted string""";
+SELECT r'''triple single quote''';
+SELECT rb'''triple single quote''';
+SELECT rB'''triple single quote''';
+SELECT R'''triple single quote''';
+SELECT Rb'''triple single quote''';
+SELECT RB'''triple single quote''';
+SELECT r"""triple double quote""";
+SELECT rb"""triple double quote""";
+SELECT rB"""triple double quote""";
+SELECT R"""triple double quote""";
+SELECT Rb"""triple double quote""";
+SELECT RB"""triple double quote""";
+SELECT RBr"""triple double quote""";
+SELECT RBb"""triple double quote""";
+SELECT rr"invalid raw single quoted string";
+SELECT rr"""invalid raw single quoted string""";
+SELECT rr'invalid raw single quoted string';
+
+---tokens---
+'BEGIN'       Keyword
+' '           Text.Whitespace
+'TRANSACTION' Keyword
+'\n'          Text.Whitespace
+
+'CREATE'      Keyword
+' '           Text.Whitespace
+'TEMP'        Keyword
+' '           Text.Whitespace
+'TABLE'       Keyword
+' '           Text.Whitespace
+'Flights'     Name
+'('           Punctuation
+'total'       Name
+' '           Text.Whitespace
+'INT64'       Keyword.Type
+')'           Punctuation
+'  '          Text.Whitespace
+'AS'          Keyword
+' '           Text.Whitespace
+'SELECT'      Keyword
+' '           Text.Whitespace
+'*'           Error
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'UNNEST'      Keyword
+'('           Punctuation
+'['           Error
+'10'          Literal.Number.Integer
+','           Punctuation
+'23'          Literal.Number.Integer
+','           Punctuation
+'3'           Literal.Number.Integer
+','           Punctuation
+'14'          Literal.Number.Integer
+','           Punctuation
+'55'          Literal.Number.Integer
+']'           Error
+')'           Punctuation
+' '           Text.Whitespace
+'AS'          Keyword
+' '           Text.Whitespace
+'a'           Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'*'           Error
+','           Punctuation
+' '           Text.Whitespace
+'SPLIT_SUBSTR' Name.Function
+'('           Punctuation
+"'"           Literal.String.Single
+'blah'        Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+'3'           Literal.Number.Integer
+'||'          Operator
+'4'           Literal.Number.Integer
+')'           Punctuation
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'Flights'     Name
+';'           Punctuation
+'\n\n'        Text.Whitespace
+
+'## function calls and builtins' Comment.Single
+'\n\n'        Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'abs'         Name.Function
+'('           Punctuation
+'1.5'         Literal.Number.Float
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'CREATE'      Keyword
+' '           Text.Whitespace
+'OR'          Keyword
+' '           Text.Whitespace
+'REPLACE'     Keyword
+' '           Text.Whitespace
+'my_table'    Name
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'REPLACE'     Keyword
+'('           Punctuation
+"'"           Literal.String.Single
+'good show'   Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'good'        Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'great'       Literal.String.Single
+"'"           Literal.String.Single
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'LPAD'        Name.Function
+'('           Punctuation
+'"'           Literal.String.Double
+'hello'       Literal.String.Double
+'"'           Literal.String.Double
+','           Punctuation
+' '           Text.Whitespace
+'10'          Literal.Number.Integer
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'D3A_COUNT.TO_HLL' Name.Function
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'KLL_QUANTILES' Name
+'.'           Punctuation
+'MERGE_UINT64' Name
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'MyNonBuiltinFunction' Name
+'('           Punctuation
+'an_arg'      Name
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'MyNonBuiltinFunction' Name
+'('           Punctuation
+'('           Punctuation
+'SELECT'      Keyword
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+')'           Punctuation
+','           Punctuation
+' '           Text.Whitespace
+'an_arg'      Name
+')'           Punctuation
+','           Punctuation
+' '           Text.Whitespace
+'Baz'         Name
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'MyNonBuiltinFunction' Name
+'('           Punctuation
+"'"           Literal.String.Single
+')'           Literal.String.Single
+"'"           Literal.String.Single
+')'           Punctuation
+','           Punctuation
+' '           Text.Whitespace
+'Baz'         Name
+'('           Punctuation
+'an_arg'      Name
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'('           Punctuation
+'1'           Literal.Number.Integer
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'DEFINE'      Keyword
+' '           Text.Whitespace
+'TABLE'       Keyword
+' '           Text.Whitespace
+'Foo'         Name
+' '           Text.Whitespace
+'('           Punctuation
+'options'     Keyword
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'DEFINE'      Keyword
+' '           Text.Whitespace
+'MACRO'       Keyword
+' '           Text.Whitespace
+'FOO'         Name
+' '           Text.Whitespace
+'the'         Name
+' '           Text.Whitespace
+'body'        Name
+' '           Text.Whitespace
+'of'          Keyword
+' '           Text.Whitespace
+'the'         Name
+' '           Text.Whitespace
+'macro'       Keyword
+' '           Text.Whitespace
+'with'        Keyword
+' '           Text.Whitespace
+'first'       Keyword
+' '           Text.Whitespace
+'arg'         Name
+' '           Text.Whitespace
+'$1'          Name
+' '           Text.Whitespace
+'and'         Keyword
+' '           Text.Whitespace
+'second '     Keyword
+'arg'         Name
+' '           Text.Whitespace
+'$2'          Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'$mymacro'    Name
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'my_table'    Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'A macro inside a string: $mymacro' Literal.String.Double
+'"'           Literal.String.Double
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'my_table'    Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'A macro inside a string: $mymacro' Literal.String.Single
+"'"           Literal.String.Single
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'my_table'    Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'A macro inside a string: $mymacro' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'my_table'    Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rb'          Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'A macro inside a string: $mymacro(' Literal.String.Double
+'"'           Literal.String.Double
+'arg1'        Name
+'"'           Literal.String.Double
+', '          Literal.String.Double
+'"'           Literal.String.Double
+'arg2'        Name
+'"'           Literal.String.Double
+')'           Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'my_table'    Name
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'creation'    Name
+','           Punctuation
+' '           Text.Whitespace
+'latency'     Name
+' '           Text.Whitespace
+'from'        Keyword
+' '           Text.Whitespace
+'my_table_$'  Name
+'{'           Error
+'YYYYMMDD'    Name
+'+'           Operator
+'1'           Literal.Number.Integer
+'}'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'creation'    Name
+','           Punctuation
+' '           Text.Whitespace
+'latency'     Name
+' '           Text.Whitespace
+'from'        Keyword
+' '           Text.Whitespace
+'my_table_$'  Name
+'{'           Error
+'YYYY'        Name
+'+'           Operator
+'2'           Literal.Number.Integer
+'}'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'creation'    Name
+','           Punctuation
+' '           Text.Whitespace
+'latency'     Name
+' '           Text.Whitespace
+'from'        Keyword
+' '           Text.Whitespace
+'my_table_$'  Name
+'{'           Error
+'YYYYMMDD'    Name
+'-'           Operator
+'3'           Literal.Number.Integer
+'}'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'creation'    Name
+','           Punctuation
+' '           Text.Whitespace
+'latency'     Name
+' '           Text.Whitespace
+'from'        Keyword
+' '           Text.Whitespace
+'my_table_$'  Name
+'{'           Error
+'DD'          Name
+'}'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'This is the day of the month ${DD}. QED.' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'*'           Error
+' '           Text.Whitespace
+'from'        Keyword
+' '           Text.Whitespace
+'logs'        Name
+' '           Text.Whitespace
+'limit'       Keyword
+' '           Text.Whitespace
+'$'           Name
+'{'           Error
+'LimitCount'  Name
+'}'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+' '           Text.Whitespace
+'='           Error
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+','           Punctuation
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+' '           Text.Whitespace
+'!'           Error
+'='           Error
+' '           Text.Whitespace
+'42'          Literal.Number.Integer
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+' '           Text.Whitespace
+'>'           Error
+' '           Text.Whitespace
+'2'           Literal.Number.Integer
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'LOAD'        Keyword
+' '           Text.Whitespace
+'my'          Name
+'.'           Punctuation
+'script'      Name
+';'           Punctuation
+'\n\n'        Text.Whitespace
+
+'## Ambiguous keywords, either type or function or keywords.' Comment.Single
+'\n\n'        Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'JSON'        Keyword.Type
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'{"name": "Alice", "age": 30}' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'IF'          Keyword
+'('           Punctuation
+'"'           Literal.String.Double
+'%.1f'        Literal.String.Double
+'"'           Literal.String.Double
+','           Punctuation
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+'/'           Operator
+'3'           Literal.Number.Integer
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'CREATE'      Keyword
+' '           Text.Whitespace
+'TABLE'       Keyword
+' '           Text.Whitespace
+'IF'          Keyword
+' '           Text.Whitespace
+'NOT'         Keyword
+' '           Text.Whitespace
+'EXISTS'      Keyword
+' '           Text.Whitespace
+'my_table'    Name
+' '           Text.Whitespace
+'['           Error
+'('           Punctuation
+'my_column'   Name
+' '           Text.Whitespace
+'int64'       Keyword.Type
+')'           Punctuation
+']'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'EXTRACT'     Keyword
+'('           Punctuation
+'DAY'         Keyword
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'DATE'        Keyword.Type
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'2013-12-25'  Literal.String.Single
+"'"           Literal.String.Single
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'CAST'        Keyword
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text.Whitespace
+'AS'          Keyword
+' '           Text.Whitespace
+'BOOL'        Keyword.Type
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'element'     Name
+' '           Text.Whitespace
+'from'        Keyword
+' '           Text.Whitespace
+'UNNEST'      Keyword
+'('           Punctuation
+'['           Error
+'1'           Literal.Number.Integer
+','           Punctuation
+' '           Text.Whitespace
+'2'           Literal.Number.Integer
+','           Punctuation
+' '           Text.Whitespace
+'3'           Literal.Number.Integer
+']'           Error
+' '           Text.Whitespace
+'as'          Keyword
+' '           Text.Whitespace
+'element'     Name
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'select'      Keyword
+' '           Text.Whitespace
+'['           Error
+'1'           Literal.Number.Integer
+','           Punctuation
+' '           Text.Whitespace
+'2'           Literal.Number.Integer
+','           Punctuation
+' '           Text.Whitespace
+'3'           Literal.Number.Integer
+']'           Error
+'['           Error
+'OFFSET'      Keyword
+'('           Punctuation
+'0'           Literal.Number.Integer
+')'           Punctuation
+']'           Error
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'*'           Error
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'UNNEST'      Keyword
+'('           Punctuation
+'['           Error
+"'"           Literal.String.Single
+'a'           Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'b'           Literal.String.Single
+"'"           Literal.String.Single
+']'           Error
+')'           Punctuation
+' '           Text.Whitespace
+'AS'          Keyword
+' '           Text.Whitespace
+'letter'      Name
+'\n'          Text.Whitespace
+
+'ORDER'       Keyword
+' '           Text.Whitespace
+'BY'          Keyword
+' '           Text.Whitespace
+'letter'      Name
+' '           Text.Whitespace
+'ASC'         Keyword
+' '           Text.Whitespace
+'LIMIT'       Keyword
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+' '           Text.Whitespace
+'OFFSET'      Keyword
+' '           Text.Whitespace
+'1'           Literal.Number.Integer
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'*'           Error
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'Produce'     Name
+' '           Text.Whitespace
+'PIVOT'       Keyword
+'('           Punctuation
+'SUM'         Name.Function
+'('           Punctuation
+'sales'       Name
+')'           Punctuation
+' '           Text.Whitespace
+'FOR'         Keyword
+' '           Text.Whitespace
+'quarter'     Keyword
+' '           Text.Whitespace
+'IN'          Keyword
+' '           Text.Whitespace
+'('           Punctuation
+"'"           Literal.String.Single
+'Q1'          Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'Q2'          Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'Q3'          Literal.String.Single
+"'"           Literal.String.Single
+','           Punctuation
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'Q4'          Literal.String.Single
+"'"           Literal.String.Single
+')'           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'*'           Error
+' '           Text.Whitespace
+'FROM'        Keyword
+' '           Text.Whitespace
+'Produce'     Name
+' '           Text.Whitespace
+'UNPIVOT'     Keyword
+'('           Punctuation
+'sales'       Name
+' '           Text.Whitespace
+'FOR'         Keyword
+' '           Text.Whitespace
+'quarter'     Keyword
+' '           Text.Whitespace
+'IN'          Keyword
+' '           Text.Whitespace
+'('           Punctuation
+'Q1'          Name
+','           Punctuation
+' '           Text.Whitespace
+'Q2'          Name
+','           Punctuation
+' '           Text.Whitespace
+'Q3'          Name
+','           Punctuation
+' '           Text.Whitespace
+'Q4'          Name
+')'           Punctuation
+')'           Punctuation
+'\n\n'        Text.Whitespace
+
+'## comments' Comment.Single
+'\n\n'        Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'TRUE'        Name.Constant
+';'           Punctuation
+'  '          Text.Whitespace
+'-- double-dash comment' Comment.Single
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'TRUE'        Name.Constant
+';'           Punctuation
+'  '          Text.Whitespace
+'# hash comment' Comment.Single
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'TRUE'        Name.Constant
+';'           Punctuation
+'  '          Text.Whitespace
+'/'           Error
+'/'           Error
+' '           Text.Whitespace
+'this'        Name
+' '           Text.Whitespace
+'is'          Keyword
+' '           Text.Whitespace
+'a'           Name
+' '           Text.Whitespace
+'comment'     Name
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'/*'          Comment.Multiline
+' this is block comment ' Comment.Multiline
+'*/'          Comment.Multiline
+' '           Text.Whitespace
+'TRUE'        Name.Constant
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'# Bracket comments in stock SQL VS Code config are not supported in GoogleSQL' Comment.Single
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'%'           Error
+'{'           Error
+'bracket'     Name
+' '           Text.Whitespace
+'comment'     Name
+'}'           Error
+';'           Punctuation
+'\n\n'        Text.Whitespace
+
+'## embedding JS' Comment.Single
+'\n\n'        Text.Whitespace
+
+'CREATE'      Keyword
+' '           Text.Whitespace
+'FUNCTION'    Keyword
+' '           Text.Whitespace
+'f'           Name
+'('           Punctuation
+'x'           Name
+' '           Text.Whitespace
+'INT'         Name
+','           Punctuation
+' '           Text.Whitespace
+'y'           Name
+' '           Text.Whitespace
+'INT'         Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'RETURNS'     Keyword
+' '           Text.Whitespace
+'DOUBLE'      Keyword.Type
+' '           Text.Whitespace
+'LANGUAGE'    Keyword
+' '           Text.Whitespace
+'js'          Name
+' '           Text.Whitespace
+'AS'          Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'return (x*y);' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n\n'        Text.Whitespace
+
+'## embedding Lua' Comment.Single
+'\n\n'        Text.Whitespace
+
+'CREATE'      Keyword
+' '           Text.Whitespace
+'FUNCTION'    Keyword
+' '           Text.Whitespace
+'f'           Name
+'('           Punctuation
+'x'           Name
+' '           Text.Whitespace
+'INT'         Name
+','           Punctuation
+' '           Text.Whitespace
+'y'           Name
+' '           Text.Whitespace
+'INT'         Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'RETURNS'     Keyword
+' '           Text.Whitespace
+'DOUBLE'      Keyword.Type
+' '           Text.Whitespace
+'LANGUAGE'    Keyword
+' '           Text.Whitespace
+'lua'         Name
+' '           Text.Whitespace
+'AS'          Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'return (x*y);' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n\n'        Text.Whitespace
+
+'### strings' Comment.Single
+'\n\n'        Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'Hi !'        Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'Hi !'        Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'a tab '      Literal.String.Single
+'\\t'         Literal.String.Escape
+' in a single-quoted string' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'a tab '      Literal.String.Double
+'\\t'         Literal.String.Escape
+' in a single-quoted string' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'string with a # sign' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'string with a # sign' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'a string with a ' Literal.String.Single
+"\\'"         Literal.String.Escape
+' single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'a string with a ' Literal.String.Double
+'\\"'         Literal.String.Escape
+' double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+'triple single quote no word boundary' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'"'           Literal.String.Double
+'triple single quote no word boundary' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote no word boundary' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'b'           Name
+"'"           Literal.String.Single
+'binary single quoted string' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'B'           Name
+"'"           Literal.String.Single
+'binary single quoted string' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'b'           Name
+'"'           Literal.String.Double
+'binary double quoted string' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'B'           Name
+'"'           Literal.String.Double
+'binary double quoted string' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'b'           Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'B'           Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'b'           Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'B'           Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+'"'           Literal.String.Double
+'raw single quoted string' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+'"'           Literal.String.Double
+'double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rb'          Name
+'"'           Literal.String.Double
+'double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rB'          Name
+'"'           Literal.String.Double
+'double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'R'           Name
+'"'           Literal.String.Double
+'double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'Rb'          Name
+'"'           Literal.String.Double
+'double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'RB'          Name
+'"'           Literal.String.Double
+'double quote' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+"'"           Literal.String.Single
+'single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rb'          Name
+"'"           Literal.String.Single
+'single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rB'          Name
+"'"           Literal.String.Single
+'single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'R'           Name
+"'"           Literal.String.Single
+'single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'Rb'          Name
+"'"           Literal.String.Single
+'single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'RB'          Name
+"'"           Literal.String.Single
+'single quote' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+"'"           Literal.String.Single
+'single quote with an uninterpreted ' Literal.String.Single
+"\\'"         Literal.String.Escape
+' escape'     Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'b'           Name
+"'"           Literal.String.Single
+'single quote with an *interpreted* ' Literal.String.Single
+"\\'"         Literal.String.Escape
+' escape'     Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'raw single quoted string' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rb'          Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rB'          Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'R'           Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'Rb'          Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'RB'          Name
+"'"           Literal.String.Single
+"''"          Literal.String.Escape
+'triple single quote' Literal.String.Single
+"''"          Literal.String.Escape
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'r'           Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rb'          Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rB'          Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'R'           Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'Rb'          Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'RB'          Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'RBr'         Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'RBb'         Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'triple double quote' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rr'          Name
+'"'           Literal.String.Double
+'invalid raw single quoted string' Literal.String.Double
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rr'          Name
+'"'           Literal.String.Double
+'""'          Literal.String.Escape
+'invalid raw single quoted string' Literal.String.Double
+'""'          Literal.String.Escape
+'"'           Literal.String.Double
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'SELECT'      Keyword
+' '           Text.Whitespace
+'rr'          Name
+"'"           Literal.String.Single
+'invalid raw single quoted string' Literal.String.Single
+"'"           Literal.String.Single
+';'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/rego/test_assignment_and_equality.txt b/tests/snippets/rego/test_assignment_and_equality.txt
new file mode 100644
index 00000000..092ddd47
--- /dev/null
+++ b/tests/snippets/rego/test_assignment_and_equality.txt
@@ -0,0 +1,94 @@
+---input---
+package example
+
+# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/
+
+# assign variable x to value of field foo.bar.baz in input
+x := input.foo.bar.baz
+
+# check if variable x has same value as variable y
+x == y
+
+# check if variable x is a set containing "foo" and "bar"
+x == {"foo", "bar"}
+
+# OR
+
+{"foo", "bar"} == x
+
+---tokens---
+'package'     Keyword
+' '           Text
+'example'     Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# assign variable x to value of field foo.bar.baz in input' Comment.Single
+'\n'          Text.Whitespace
+
+'x'           Name
+' '           Text
+':='          Operator
+' '           Text
+'input'       Name.Builtin
+'.'           Punctuation
+'foo'         Name
+'.'           Punctuation
+'bar'         Name
+'.'           Punctuation
+'baz'         Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if variable x has same value as variable y' Comment.Single
+'\n'          Text.Whitespace
+
+'x'           Name
+' '           Text
+'=='          Operator
+' '           Text
+'y'           Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if variable x is a set containing "foo" and "bar"' Comment.Single
+'\n'          Text.Whitespace
+
+'x'           Name
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'"foo"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'"bar"'       Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# OR'        Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"foo"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'"bar"'       Literal.String.Double
+'}'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'x'           Name
+'\n'          Text.Whitespace
diff --git a/tests/snippets/rego/test_builtin_functions.txt b/tests/snippets/rego/test_builtin_functions.txt
new file mode 100644
index 00000000..0087b5d7
--- /dev/null
+++ b/tests/snippets/rego/test_builtin_functions.txt
@@ -0,0 +1,1880 @@
+---input---
+package test_builtins
+
+# Comparison
+test_comparison {
+    1 == 1
+    1 != 2
+    2 > 1
+    2 >= 2
+    1 < 2
+    1 <= 1
+}
+
+# Numbers
+test_numbers {
+    abs(-5) == 5
+    ceil(3.7) == 4
+    floor(3.7) == 3
+    round(3.5) == 4
+    numbers.range(1, 3) == [1, 2, 3]
+    7 / 2 == 3.5
+    7 % 3 == 1
+}
+
+# Aggregates
+test_aggregates {
+    count([1, 2, 3]) == 3
+    sum([1, 2, 3]) == 6
+    product([2, 3, 4]) == 24
+    max([1, 5, 3]) == 5
+    min([1, 5, 3]) == 1
+    sort([3, 1, 2]) == [1, 2, 3]
+}
+
+# Arrays
+test_arrays {
+    array.concat([1, 2], [3, 4]) == [1, 2, 3, 4]
+    array.slice([1, 2, 3, 4], 1, 3) == [2, 3]
+    array.reverse([1, 2, 3]) == [3, 2, 1]
+}
+
+# Sets
+test_sets {
+    {1, 2} & {2, 3} == {2}
+    {1, 2} | {2, 3} == {1, 2, 3}
+    {1, 2, 3} - {2} == {1, 3}
+    intersection({{1, 2}, {2, 3}}) == {2}
+    union({{1, 2}, {2, 3}}) == {1, 2, 3}
+}
+
+# Objects
+test_objects {
+    object.get({"a": 1, "b": 2}, "a", 0) == 1
+    object.get({"a": 1, "b": 2}, "c", 0) == 0
+    object.remove({"a": 1, "b": 2}, ["a"]) == {"b": 2}
+    object.union({"a": 1}, {"b": 2}) == {"a": 1, "b": 2}
+}
+
+# Strings
+test_strings {
+    contains("abcde", "bc")
+    startswith("abcde", "ab")
+    endswith("abcde", "de")
+    lower("AbC") == "abc"
+    upper("aBc") == "ABC"
+    trim("  abc  ") == "abc"
+    split("a,b,c", ",") == ["a", "b", "c"]
+    replace("aabbcc", "b", "d") == "aaddcc"
+    sprintf("%s %d", ["foo", 5]) == "foo 5"
+}
+
+# Regex
+test_regex {
+    regex.match("^a[bc]+d$", "abcd")
+    regex.split(":", "a:b:c") == ["a", "b", "c"]
+    regex.find_n("[a-z]", "a1b2c3", -1) == ["a", "b", "c"]
+}
+
+# Glob
+test_glob {
+    glob.match("*.txt", ["."], "file.txt")
+    not glob.match("*.txt", ["."], "file.jpg")
+}
+
+# Type checks
+test_types {
+    is_number(5)
+    is_string("abc")
+    is_boolean(true)
+    is_array([1, 2, 3])
+    is_set({1, 2, 3})
+    is_object({"a": 1})
+    is_null(null)
+    type_name(5) == "number"
+}
+
+# Encoding
+test_encoding {
+    base64.encode("hello") == "aGVsbG8="
+    base64.decode("aGVsbG8=") == "hello"
+    json.marshal({"a": 1}) == "{\"a\":1}"
+    json.unmarshal("{\"a\":1}") == {"a": 1}
+}
+
+# Time
+test_time {
+    time.now_ns() > 0
+    time.parse_duration_ns("1h") == 3600000000000
+    parsed := time.parse_rfc3339_ns("2006-01-02T15:04:05Z")
+    [year, month, day] := time.date(parsed)
+    year == 2006
+    month == 1
+    day == 2
+}
+
+# Crypto
+test_crypto {
+    crypto.md5("hello") == "5d41402abc4b2a76b9719d911017c592"
+    crypto.sha1("hello") == "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d"
+    crypto.sha256("hello") == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
+}
+
+# Net
+test_net {
+    net.cidr_contains("192.168.1.0/24", "192.168.1.100")
+    not net.cidr_contains("192.168.1.0/24", "192.168.2.100")
+}
+
+# Graphs
+test_graphs {
+    graph := {"a": ["b", "c"], "b": ["d"], "c": ["d"], "d": []}
+    graph.reachable(graph, {"a"}) == {"a", "b", "c", "d"}
+}
+
+# HTTP and JWTs (mocked)
+mock_http_send(request) = http_response if {
+    request.url == "https://example.com"
+    http_response := {
+        "status": "200 OK",
+        "status_code": 200,
+        "body": "{\"key\": \"value\"}",
+    }
+}
+
+test_http_and_jwt {
+    http.send({"method": "GET", "url": "https://example.com"}) == mock_http_send({"method": "GET", "url": "https://example.com"})
+    io.jwt.encode_sign({"alg": "HS256"}, {"sub": "1234567890"}, {"kty": "oct", "k": "secret"})
+}
+
+# UUID
+test_uuid {
+    uuid := uuid.rfc4122("test")
+    startswith(uuid, "")  # Always true, just to use the uuid variable
+    uuid2 := uuid.rfc4122("test")
+    uuid == uuid2  # Consistent for the same input
+}
+
+# Rego
+test_rego {
+    parsed := rego.parse_module("test.rego", "package test")
+    parsed.package.path[0].value == "test"
+}
+
+# OPA Runtime
+test_opa_runtime {
+    runtime := opa.runtime()
+    is_object(runtime)
+}
+
+---tokens---
+'package'     Keyword
+' '           Text
+'test_builtins' Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Comparison' Comment.Single
+'\n'          Text.Whitespace
+
+'test_comparison' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'1'           Literal.Number
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'1'           Literal.Number
+' '           Text
+'!='          Operator
+' '           Text
+'2'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'2'           Literal.Number
+' '           Text
+'>'           Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'2'           Literal.Number
+' '           Text
+'>='          Operator
+' '           Text
+'2'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'1'           Literal.Number
+' '           Text
+'<'           Operator
+' '           Text
+'2'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'1'           Literal.Number
+' '           Text
+'<='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Numbers'   Comment.Single
+'\n'          Text.Whitespace
+
+'test_numbers' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'abs'         Name
+'('           Punctuation
+'-5'          Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'5'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'ceil'        Name
+'('           Punctuation
+'3.7'         Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'4'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'floor'       Name
+'('           Punctuation
+'3.7'         Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'3'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'round'       Name
+'('           Punctuation
+'3.5'         Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'4'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'numbers'     Name
+'.'           Punctuation
+'range'       Name
+'('           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'7'           Literal.Number
+' '           Text
+'/'           Operator
+' '           Text
+'2'           Literal.Number
+' '           Text
+'=='          Operator
+' '           Text
+'3.5'         Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'7'           Literal.Number
+' '           Text
+'%'           Operator
+' '           Text
+'3'           Literal.Number
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Aggregates' Comment.Single
+'\n'          Text.Whitespace
+
+'test_aggregates' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'count'       Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'3'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'sum'         Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'6'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'product'     Name
+'('           Punctuation
+'['           Punctuation
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+','           Punctuation
+' '           Text
+'4'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'24'          Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'max'         Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'5'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'5'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'min'         Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'5'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'sort'        Name
+'('           Punctuation
+'['           Punctuation
+'3'           Literal.Number
+','           Punctuation
+' '           Text
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Arrays'    Comment.Single
+'\n'          Text.Whitespace
+
+'test_arrays' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'array'       Name
+'.'           Punctuation
+'concat'      Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+']'           Punctuation
+','           Punctuation
+' '           Text
+'['           Punctuation
+'3'           Literal.Number
+','           Punctuation
+' '           Text
+'4'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+','           Punctuation
+' '           Text
+'4'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'array'       Name
+'.'           Punctuation
+'slice'       Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+','           Punctuation
+' '           Text
+'4'           Literal.Number
+']'           Punctuation
+','           Punctuation
+' '           Text
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'array'       Name
+'.'           Punctuation
+'reverse'     Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'3'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'1'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Sets'      Comment.Single
+'\n'          Text.Whitespace
+
+'test_sets'   Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+' '           Text
+'&'           Operator
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+' '           Text
+'|'           Operator
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+' '           Text
+'-'           Operator
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+'}'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'intersection' Name
+'('           Punctuation
+'{'           Punctuation
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'union'       Name
+'('           Punctuation
+'{'           Punctuation
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'{'           Punctuation
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Objects'   Comment.Single
+'\n'          Text.Whitespace
+
+'test_objects' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'object'      Name
+'.'           Punctuation
+'get'         Name
+'('           Punctuation
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'0'           Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'object'      Name
+'.'           Punctuation
+'get'         Name
+'('           Punctuation
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'0'           Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'0'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'object'      Name
+'.'           Punctuation
+'remove'      Name
+'('           Punctuation
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'['           Punctuation
+'"a"'         Literal.String.Double
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'object'      Name
+'.'           Punctuation
+'union'       Name
+'('           Punctuation
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'{'           Punctuation
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'2'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Strings'   Comment.Single
+'\n'          Text.Whitespace
+
+'test_strings' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'contains'    Keyword
+'('           Punctuation
+'"abcde"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'"bc"'        Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'startswith'  Name
+'('           Punctuation
+'"abcde"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'"ab"'        Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'endswith'    Name
+'('           Punctuation
+'"abcde"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'"de"'        Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'lower'       Name
+'('           Punctuation
+'"AbC"'       Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"abc"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'upper'       Name
+'('           Punctuation
+'"aBc"'       Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"ABC"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'trim'        Name
+'('           Punctuation
+'"  abc  "'   Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"abc"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'split'       Name
+'('           Punctuation
+'"a,b,c"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'","'         Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'replace'     Name
+'('           Punctuation
+'"aabbcc"'    Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"d"'         Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"aaddcc"'    Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'sprintf'     Name
+'('           Punctuation
+'"%s %d"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'['           Punctuation
+'"foo"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'5'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"foo 5"'     Literal.String.Double
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Regex'     Comment.Single
+'\n'          Text.Whitespace
+
+'test_regex'  Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'regex'       Name
+'.'           Punctuation
+'match'       Name
+'('           Punctuation
+'"^a[bc]+d$"' Literal.String.Double
+','           Punctuation
+' '           Text
+'"abcd"'      Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'regex'       Name
+'.'           Punctuation
+'split'       Name
+'('           Punctuation
+'":"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"a:b:c"'     Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'regex'       Name
+'.'           Punctuation
+'find_n'      Name
+'('           Punctuation
+'"[a-z]"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'"a1b2c3"'    Literal.String.Double
+','           Punctuation
+' '           Text
+'-1'          Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'['           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Glob'      Comment.Single
+'\n'          Text.Whitespace
+
+'test_glob'   Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'glob'        Name
+'.'           Punctuation
+'match'       Name
+'('           Punctuation
+'"*.txt"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'['           Punctuation
+'"."'         Literal.String.Double
+']'           Punctuation
+','           Punctuation
+' '           Text
+'"file.txt"'  Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'not'         Keyword
+' '           Text
+'glob'        Name
+'.'           Punctuation
+'match'       Name
+'('           Punctuation
+'"*.txt"'     Literal.String.Double
+','           Punctuation
+' '           Text
+'['           Punctuation
+'"."'         Literal.String.Double
+']'           Punctuation
+','           Punctuation
+' '           Text
+'"file.jpg"'  Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Type checks' Comment.Single
+'\n'          Text.Whitespace
+
+'test_types'  Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_number'   Name
+'('           Punctuation
+'5'           Literal.Number
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_string'   Name
+'('           Punctuation
+'"abc"'       Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_boolean'  Name
+'('           Punctuation
+'true'        Keyword
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_array'    Name
+'('           Punctuation
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_set'      Name
+'('           Punctuation
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_object'   Name
+'('           Punctuation
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+'}'           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_null'     Name
+'('           Punctuation
+'null'        Keyword
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'type_name'   Name
+'('           Punctuation
+'5'           Literal.Number
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"number"'    Literal.String.Double
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Encoding'  Comment.Single
+'\n'          Text.Whitespace
+
+'test_encoding' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'base64'      Name
+'.'           Punctuation
+'encode'      Name
+'('           Punctuation
+'"hello"'     Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"aGVsbG8="'  Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'base64'      Name
+'.'           Punctuation
+'decode'      Name
+'('           Punctuation
+'"aGVsbG8="'  Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"hello"'     Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'json'        Name
+'.'           Punctuation
+'marshal'     Name
+'('           Punctuation
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"{\\"a\\":1}"' Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'json'        Name
+'.'           Punctuation
+'unmarshal'   Name
+'('           Punctuation
+'"{\\"a\\":1}"' Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'1'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Time'      Comment.Single
+'\n'          Text.Whitespace
+
+'test_time'   Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'time'        Name
+'.'           Punctuation
+'now_ns'      Name
+'('           Punctuation
+')'           Punctuation
+' '           Text
+'>'           Operator
+' '           Text
+'0'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'time'        Name
+'.'           Punctuation
+'parse_duration_ns' Name
+'('           Punctuation
+'"1h"'        Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'3600000000000' Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'parsed'      Name
+' '           Text
+':='          Operator
+' '           Text
+'time'        Name
+'.'           Punctuation
+'parse_rfc3339_ns' Name
+'('           Punctuation
+'"2006-01-02T15:04:05Z"' Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'['           Punctuation
+'year'        Name
+','           Punctuation
+' '           Text
+'month'       Name
+','           Punctuation
+' '           Text
+'day'         Name
+']'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'time'        Name
+'.'           Punctuation
+'date'        Name
+'('           Punctuation
+'parsed'      Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'year'        Name
+' '           Text
+'=='          Operator
+' '           Text
+'2006'        Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'month'       Name
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'    '        Text
+'day'         Name
+' '           Text
+'=='          Operator
+' '           Text
+'2'           Literal.Number
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Crypto'    Comment.Single
+'\n'          Text.Whitespace
+
+'test_crypto' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'crypto'      Name
+'.'           Punctuation
+'md5'         Name
+'('           Punctuation
+'"hello"'     Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"5d41402abc4b2a76b9719d911017c592"' Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'crypto'      Name
+'.'           Punctuation
+'sha1'        Name
+'('           Punctuation
+'"hello"'     Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d"' Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'crypto'      Name
+'.'           Punctuation
+'sha256'      Name
+'('           Punctuation
+'"hello"'     Literal.String.Double
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"' Literal.String.Double
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Net'       Comment.Single
+'\n'          Text.Whitespace
+
+'test_net'    Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'net'         Name
+'.'           Punctuation
+'cidr_contains' Name
+'('           Punctuation
+'"192.168.1.0/24"' Literal.String.Double
+','           Punctuation
+' '           Text
+'"192.168.1.100"' Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'not'         Keyword
+' '           Text
+'net'         Name
+'.'           Punctuation
+'cidr_contains' Name
+'('           Punctuation
+'"192.168.1.0/24"' Literal.String.Double
+','           Punctuation
+' '           Text
+'"192.168.2.100"' Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Graphs'    Comment.Single
+'\n'          Text.Whitespace
+
+'test_graphs' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'graph'       Name
+' '           Text
+':='          Operator
+' '           Text
+'{'           Punctuation
+'"a"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'['           Punctuation
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+']'           Punctuation
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'['           Punctuation
+'"d"'         Literal.String.Double
+']'           Punctuation
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'['           Punctuation
+'"d"'         Literal.String.Double
+']'           Punctuation
+','           Punctuation
+' '           Text
+'"d"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'['           Punctuation
+']'           Punctuation
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'graph'       Name
+'.'           Punctuation
+'reachable'   Name
+'('           Punctuation
+'graph'       Name
+','           Punctuation
+' '           Text
+'{'           Punctuation
+'"a"'         Literal.String.Double
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'{'           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"d"'         Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# HTTP and JWTs (mocked)' Comment.Single
+'\n'          Text.Whitespace
+
+'mock_http_send' Name
+'('           Punctuation
+'request'     Name
+')'           Punctuation
+' '           Text
+'='           Operator
+' '           Text
+'http_response' Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'request'     Name
+'.'           Punctuation
+'url'         Name
+' '           Text
+'=='          Operator
+' '           Text
+'"https://example.com"' Literal.String.Double
+'\n'          Text.Whitespace
+
+'    '        Text
+'http_response' Name
+' '           Text
+':='          Operator
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text
+'"status"'    Literal.String.Double
+':'           Punctuation
+' '           Text
+'"200 OK"'    Literal.String.Double
+','           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text
+'"status_code"' Literal.String.Double
+':'           Punctuation
+' '           Text
+'200'         Literal.Number
+','           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text
+'"body"'      Literal.String.Double
+':'           Punctuation
+' '           Text
+'"{\\"key\\": \\"value\\"}"' Literal.String.Double
+','           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'test_http_and_jwt' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'http'        Name
+'.'           Punctuation
+'send'        Name
+'('           Punctuation
+'{'           Punctuation
+'"method"'    Literal.String.Double
+':'           Punctuation
+' '           Text
+'"GET"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'"url"'       Literal.String.Double
+':'           Punctuation
+' '           Text
+'"https://example.com"' Literal.String.Double
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'mock_http_send' Name
+'('           Punctuation
+'{'           Punctuation
+'"method"'    Literal.String.Double
+':'           Punctuation
+' '           Text
+'"GET"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'"url"'       Literal.String.Double
+':'           Punctuation
+' '           Text
+'"https://example.com"' Literal.String.Double
+'}'           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'io'          Name
+'.'           Punctuation
+'jwt'         Name
+'.'           Punctuation
+'encode_sign' Name
+'('           Punctuation
+'{'           Punctuation
+'"alg"'       Literal.String.Double
+':'           Punctuation
+' '           Text
+'"HS256"'     Literal.String.Double
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'{'           Punctuation
+'"sub"'       Literal.String.Double
+':'           Punctuation
+' '           Text
+'"1234567890"' Literal.String.Double
+'}'           Punctuation
+','           Punctuation
+' '           Text
+'{'           Punctuation
+'"kty"'       Literal.String.Double
+':'           Punctuation
+' '           Text
+'"oct"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'"k"'         Literal.String.Double
+':'           Punctuation
+' '           Text
+'"secret"'    Literal.String.Double
+'}'           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# UUID'      Comment.Single
+'\n'          Text.Whitespace
+
+'test_uuid'   Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'uuid'        Name
+' '           Text
+':='          Operator
+' '           Text
+'uuid'        Name
+'.'           Punctuation
+'rfc4122'     Name
+'('           Punctuation
+'"test"'      Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'startswith'  Name
+'('           Punctuation
+'uuid'        Name
+','           Punctuation
+' '           Text
+'""'          Literal.String.Double
+')'           Punctuation
+'  '          Text
+'# Always true, just to use the uuid variable' Comment.Single
+'\n'          Text.Whitespace
+
+'    '        Text
+'uuid2'       Name
+' '           Text
+':='          Operator
+' '           Text
+'uuid'        Name
+'.'           Punctuation
+'rfc4122'     Name
+'('           Punctuation
+'"test"'      Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'uuid'        Name
+' '           Text
+'=='          Operator
+' '           Text
+'uuid2'       Name
+'  '          Text
+'# Consistent for the same input' Comment.Single
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Rego'      Comment.Single
+'\n'          Text.Whitespace
+
+'test_rego'   Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'parsed'      Name
+' '           Text
+':='          Operator
+' '           Text
+'rego'        Name
+'.'           Punctuation
+'parse_module' Name
+'('           Punctuation
+'"test.rego"' Literal.String.Double
+','           Punctuation
+' '           Text
+'"package test"' Literal.String.Double
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'parsed'      Name
+'.'           Punctuation
+'package'     Keyword
+'.'           Punctuation
+'path'        Name
+'['           Punctuation
+'0'           Literal.Number
+']'           Punctuation
+'.'           Punctuation
+'value'       Name
+' '           Text
+'=='          Operator
+' '           Text
+'"test"'      Literal.String.Double
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# OPA Runtime' Comment.Single
+'\n'          Text.Whitespace
+
+'test_opa_runtime' Name
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'runtime'     Name
+' '           Text
+':='          Operator
+' '           Text
+'opa'         Name
+'.'           Punctuation
+'runtime'     Name
+'('           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'is_object'   Name
+'('           Punctuation
+'runtime'     Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/rego/test_for_all.txt b/tests/snippets/rego/test_for_all.txt
new file mode 100644
index 00000000..da1f8237
--- /dev/null
+++ b/tests/snippets/rego/test_for_all.txt
@@ -0,0 +1,187 @@
+---input---
+package example
+
+# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/
+
+# assert no values in set match predicate
+count({x | set[x]; f(x)}) == 0
+
+# assert all values in set make function f true
+count({x | set[x]; f(x)}) == count(set)
+
+# assert no values in set make function f true (using negation and helper rule)
+not any_match
+
+# assert all values in set make function f true (using negation and helper rule)
+not any_not_match
+
+
+# with `import rego.v1`, or `import future.keywords.in` and `import future.keywords.if`
+any_match if {
+    some x in set
+    f(x)
+}
+
+any_not_match if {
+    some x in set
+    not f(x)
+}
+
+---tokens---
+'package'     Keyword
+' '           Text
+'example'     Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# assert no values in set match predicate' Comment.Single
+'\n'          Text.Whitespace
+
+'count'       Name
+'('           Punctuation
+'{'           Punctuation
+'x'           Name
+' '           Text
+'|'           Operator
+' '           Text
+'set'         Name
+'['           Punctuation
+'x'           Name
+']'           Punctuation
+';'           Punctuation
+' '           Text
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'0'           Literal.Number
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# assert all values in set make function f true' Comment.Single
+'\n'          Text.Whitespace
+
+'count'       Name
+'('           Punctuation
+'{'           Punctuation
+'x'           Name
+' '           Text
+'|'           Operator
+' '           Text
+'set'         Name
+'['           Punctuation
+'x'           Name
+']'           Punctuation
+';'           Punctuation
+' '           Text
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+'}'           Punctuation
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'count'       Name
+'('           Punctuation
+'set'         Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# assert no values in set make function f true (using negation and helper rule)' Comment.Single
+'\n'          Text.Whitespace
+
+'not'         Keyword
+' '           Text
+'any_match'   Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# assert all values in set make function f true (using negation and helper rule)' Comment.Single
+'\n'          Text.Whitespace
+
+'not'         Keyword
+' '           Text
+'any_not_match' Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1`, or `import future.keywords.in` and `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'any_match'   Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'some'        Keyword
+' '           Text
+'x'           Name
+' '           Text
+'in'          Keyword
+' '           Text
+'set'         Name
+'\n'          Text.Whitespace
+
+'    '        Text
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'any_not_match' Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'some'        Keyword
+' '           Text
+'x'           Name
+' '           Text
+'in'          Keyword
+' '           Text
+'set'         Name
+'\n'          Text.Whitespace
+
+'    '        Text
+'not'         Keyword
+' '           Text
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/rego/test_iteration.txt b/tests/snippets/rego/test_iteration.txt
new file mode 100644
index 00000000..87bfc2ed
--- /dev/null
+++ b/tests/snippets/rego/test_iteration.txt
@@ -0,0 +1,392 @@
+---input---
+package example
+
+# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/
+
+### Arrays
+
+# iterate over indices i
+arr[i]
+
+# iterate over values
+val := arr[_]
+
+# iterate over index/value pairs
+val := arr[i]
+
+# with `import rego.v1` or `import future.keywords.in`
+some val in arr    # iterate over values
+some i, _ in arr   # iterate over indices
+some i, val in arr # iterate over index/value pairs
+
+
+### Objects
+
+# iterate over keys
+obj[key]
+
+# iterate over values
+val := obj[_]
+
+# iterate over key/value pairs
+val := obj[key]
+
+# with `import rego.v1` or `import future.keywords.in`
+some val in obj      # iterate over values
+some key, _ in obj   # iterate over keys
+some key, val in obj # key/value pairs
+
+
+### Sets
+
+# iterate over values
+set[val]
+
+# with `import rego.v1` or `import future.keywords.in`
+some val in set
+
+
+### Advanced
+
+# nested: find key k whose bar.baz array index i is 7
+foo[k].bar.baz[i] == 7
+
+# simultaneous: find keys in objects foo and bar with same value
+foo[k1] == bar[k2]
+
+# simultaneous self: find 2 keys in object foo with same value
+foo[k1] == foo[k2]; k1 != k2
+
+# multiple conditions: k has same value in both conditions
+foo[k].bar.baz[i] == 7; foo[k].qux > 3
+
+---tokens---
+'package'     Keyword
+' '           Text
+'example'     Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Arrays'  Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over indices i' Comment.Single
+'\n'          Text.Whitespace
+
+'arr'         Name
+'['           Punctuation
+'i'           Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over values' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'arr'         Name
+'['           Punctuation
+'_'           Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over index/value pairs' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'arr'         Name
+'['           Punctuation
+'i'           Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.in`' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'val'         Name
+' '           Text
+'in'          Keyword
+' '           Text
+'arr'         Name
+'    '        Text
+'# iterate over values' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'i'           Name
+','           Punctuation
+' '           Text
+'_'           Name
+' '           Text
+'in'          Keyword
+' '           Text
+'arr'         Name
+'   '         Text
+'# iterate over indices' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'i'           Name
+','           Punctuation
+' '           Text
+'val'         Name
+' '           Text
+'in'          Keyword
+' '           Text
+'arr'         Name
+' '           Text
+'# iterate over index/value pairs' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Objects' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over keys' Comment.Single
+'\n'          Text.Whitespace
+
+'obj'         Name
+'['           Punctuation
+'key'         Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over values' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'obj'         Name
+'['           Punctuation
+'_'           Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over key/value pairs' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'obj'         Name
+'['           Punctuation
+'key'         Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.in`' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'val'         Name
+' '           Text
+'in'          Keyword
+' '           Text
+'obj'         Name
+'      '      Text
+'# iterate over values' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'key'         Name
+','           Punctuation
+' '           Text
+'_'           Name
+' '           Text
+'in'          Keyword
+' '           Text
+'obj'         Name
+'   '         Text
+'# iterate over keys' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'key'         Name
+','           Punctuation
+' '           Text
+'val'         Name
+' '           Text
+'in'          Keyword
+' '           Text
+'obj'         Name
+' '           Text
+'# key/value pairs' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Sets'    Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# iterate over values' Comment.Single
+'\n'          Text.Whitespace
+
+'set'         Name
+'['           Punctuation
+'val'         Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.in`' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'val'         Name
+' '           Text
+'in'          Keyword
+' '           Text
+'set'         Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Advanced' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# nested: find key k whose bar.baz array index i is 7' Comment.Single
+'\n'          Text.Whitespace
+
+'foo'         Name
+'['           Punctuation
+'k'           Name
+']'           Punctuation
+'.'           Punctuation
+'bar'         Name
+'.'           Punctuation
+'baz'         Name
+'['           Punctuation
+'i'           Name
+']'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'7'           Literal.Number
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# simultaneous: find keys in objects foo and bar with same value' Comment.Single
+'\n'          Text.Whitespace
+
+'foo'         Name
+'['           Punctuation
+'k1'          Name
+']'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'bar'         Name
+'['           Punctuation
+'k2'          Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# simultaneous self: find 2 keys in object foo with same value' Comment.Single
+'\n'          Text.Whitespace
+
+'foo'         Name
+'['           Punctuation
+'k1'          Name
+']'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'foo'         Name
+'['           Punctuation
+'k2'          Name
+']'           Punctuation
+';'           Punctuation
+' '           Text
+'k1'          Name
+' '           Text
+'!='          Operator
+' '           Text
+'k2'          Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# multiple conditions: k has same value in both conditions' Comment.Single
+'\n'          Text.Whitespace
+
+'foo'         Name
+'['           Punctuation
+'k'           Name
+']'           Punctuation
+'.'           Punctuation
+'bar'         Name
+'.'           Punctuation
+'baz'         Name
+'['           Punctuation
+'i'           Name
+']'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'7'           Literal.Number
+';'           Punctuation
+' '           Text
+'foo'         Name
+'['           Punctuation
+'k'           Name
+']'           Punctuation
+'.'           Punctuation
+'qux'         Name
+' '           Text
+'>'           Operator
+' '           Text
+'3'           Literal.Number
+'\n'          Text.Whitespace
diff --git a/tests/snippets/rego/test_lookup.txt b/tests/snippets/rego/test_lookup.txt
new file mode 100644
index 00000000..e37d8f75
--- /dev/null
+++ b/tests/snippets/rego/test_lookup.txt
@@ -0,0 +1,472 @@
+---input---
+package example
+
+# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/
+
+### Arrays
+
+# lookup value at index 0
+val := arr[0]
+
+ # check if value at index 0 is "foo"
+"foo" == arr[0]
+
+# find all indices i that have value "foo"
+"foo" == arr[i]
+
+# lookup last value
+val := arr[count(arr)-1]
+
+# with `import rego.v1` or `import future.keywords.in`
+some 0, val in arr   # lookup value at index 0
+0, "foo" in arr      # check if value at index 0 is "foo"
+some i, "foo" in arr # find all indices i that have value "foo"
+
+### Objects
+
+# lookup value for key "foo"
+val := obj["foo"]
+
+# check if value for key "foo" is "bar"
+"bar" == obj["foo"]
+
+# OR
+
+"bar" == obj.foo
+
+# check if key "foo" exists and is not false
+obj.foo
+
+# check if key assigned to variable k exists
+k := "foo"
+obj[k]
+
+# check if path foo.bar.baz exists and is not false
+obj.foo.bar.baz
+
+# check if path foo.bar.baz, foo.bar, or foo does not exist or is false
+not obj.foo.bar.baz
+
+# with `import rego.v1` or `import future.keywords.in`
+o := {"foo": false}
+# check if value exists: the expression will be true
+false in o
+# check if value for key "foo" is false
+"foo", false in o
+
+### Sets
+
+# check if "foo" belongs to the set
+a_set["foo"]
+
+# check if "foo" DOES NOT belong to the set
+not a_set["foo"]
+
+# check if the array ["a", "b", "c"] belongs to the set
+a_set[["a", "b", "c"]]
+
+# find all arrays of the form [x, "b", z] in the set
+a_set[[x, "b", z]]
+
+# with `import rego.v1` or `import future.keywords.in`
+"foo" in a_set
+not "foo" in a_set
+some ["a", "b", "c"] in a_set
+some [x, "b", z] in a_set
+
+---tokens---
+'package'     Keyword
+' '           Text
+'example'     Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Arrays'  Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# lookup value at index 0' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'arr'         Name
+'['           Punctuation
+'0'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+' '           Text
+'# check if value at index 0 is "foo"' Comment.Single
+'\n'          Text.Whitespace
+
+'"foo"'       Literal.String.Double
+' '           Text
+'=='          Operator
+' '           Text
+'arr'         Name
+'['           Punctuation
+'0'           Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# find all indices i that have value "foo"' Comment.Single
+'\n'          Text.Whitespace
+
+'"foo"'       Literal.String.Double
+' '           Text
+'=='          Operator
+' '           Text
+'arr'         Name
+'['           Punctuation
+'i'           Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# lookup last value' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'arr'         Name
+'['           Punctuation
+'count'       Name
+'('           Punctuation
+'arr'         Name
+')'           Punctuation
+'-1'          Literal.Number
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.in`' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'0'           Literal.Number
+','           Punctuation
+' '           Text
+'val'         Name
+' '           Text
+'in'          Keyword
+' '           Text
+'arr'         Name
+'   '         Text
+'# lookup value at index 0' Comment.Single
+'\n'          Text.Whitespace
+
+'0'           Literal.Number
+','           Punctuation
+' '           Text
+'"foo"'       Literal.String.Double
+' '           Text
+'in'          Keyword
+' '           Text
+'arr'         Name
+'      '      Text
+'# check if value at index 0 is "foo"' Comment.Single
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'i'           Name
+','           Punctuation
+' '           Text
+'"foo"'       Literal.String.Double
+' '           Text
+'in'          Keyword
+' '           Text
+'arr'         Name
+' '           Text
+'# find all indices i that have value "foo"' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Objects' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# lookup value for key "foo"' Comment.Single
+'\n'          Text.Whitespace
+
+'val'         Name
+' '           Text
+':='          Operator
+' '           Text
+'obj'         Name
+'['           Punctuation
+'"foo"'       Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if value for key "foo" is "bar"' Comment.Single
+'\n'          Text.Whitespace
+
+'"bar"'       Literal.String.Double
+' '           Text
+'=='          Operator
+' '           Text
+'obj'         Name
+'['           Punctuation
+'"foo"'       Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# OR'        Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'"bar"'       Literal.String.Double
+' '           Text
+'=='          Operator
+' '           Text
+'obj'         Name
+'.'           Punctuation
+'foo'         Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if key "foo" exists and is not false' Comment.Single
+'\n'          Text.Whitespace
+
+'obj'         Name
+'.'           Punctuation
+'foo'         Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if key assigned to variable k exists' Comment.Single
+'\n'          Text.Whitespace
+
+'k'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"foo"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'obj'         Name
+'['           Punctuation
+'k'           Name
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if path foo.bar.baz exists and is not false' Comment.Single
+'\n'          Text.Whitespace
+
+'obj'         Name
+'.'           Punctuation
+'foo'         Name
+'.'           Punctuation
+'bar'         Name
+'.'           Punctuation
+'baz'         Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if path foo.bar.baz, foo.bar, or foo does not exist or is false' Comment.Single
+'\n'          Text.Whitespace
+
+'not'         Keyword
+' '           Text
+'obj'         Name
+'.'           Punctuation
+'foo'         Name
+'.'           Punctuation
+'bar'         Name
+'.'           Punctuation
+'baz'         Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.in`' Comment.Single
+'\n'          Text.Whitespace
+
+'o'           Name
+' '           Text
+':='          Operator
+' '           Text
+'{'           Punctuation
+'"foo"'       Literal.String.Double
+':'           Punctuation
+' '           Text
+'false'       Keyword
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'# check if value exists: the expression will be true' Comment.Single
+'\n'          Text.Whitespace
+
+'false'       Keyword
+' '           Text
+'in'          Keyword
+' '           Text
+'o'           Name
+'\n'          Text.Whitespace
+
+'# check if value for key "foo" is false' Comment.Single
+'\n'          Text.Whitespace
+
+'"foo"'       Literal.String.Double
+','           Punctuation
+' '           Text
+'false'       Keyword
+' '           Text
+'in'          Keyword
+' '           Text
+'o'           Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Sets'    Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if "foo" belongs to the set' Comment.Single
+'\n'          Text.Whitespace
+
+'a_set'       Name
+'['           Punctuation
+'"foo"'       Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if "foo" DOES NOT belong to the set' Comment.Single
+'\n'          Text.Whitespace
+
+'not'         Keyword
+' '           Text
+'a_set'       Name
+'['           Punctuation
+'"foo"'       Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# check if the array ["a", "b", "c"] belongs to the set' Comment.Single
+'\n'          Text.Whitespace
+
+'a_set'       Name
+'['           Punctuation
+'['           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+']'           Punctuation
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# find all arrays of the form [x, "b", z] in the set' Comment.Single
+'\n'          Text.Whitespace
+
+'a_set'       Name
+'['           Punctuation
+'['           Punctuation
+'x'           Name
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'z'           Name
+']'           Punctuation
+']'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.in`' Comment.Single
+'\n'          Text.Whitespace
+
+'"foo"'       Literal.String.Double
+' '           Text
+'in'          Keyword
+' '           Text
+'a_set'       Name
+'\n'          Text.Whitespace
+
+'not'         Keyword
+' '           Text
+'"foo"'       Literal.String.Double
+' '           Text
+'in'          Keyword
+' '           Text
+'a_set'       Name
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'['           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'"c"'         Literal.String.Double
+']'           Punctuation
+' '           Text
+'in'          Keyword
+' '           Text
+'a_set'       Name
+'\n'          Text.Whitespace
+
+'some'        Keyword
+' '           Text
+'['           Punctuation
+'x'           Name
+','           Punctuation
+' '           Text
+'"b"'         Literal.String.Double
+','           Punctuation
+' '           Text
+'z'           Name
+']'           Punctuation
+' '           Text
+'in'          Keyword
+' '           Text
+'a_set'       Name
+'\n'          Text.Whitespace
diff --git a/tests/snippets/rego/test_rules.txt b/tests/snippets/rego/test_rules.txt
new file mode 100644
index 00000000..2ab0d0e3
--- /dev/null
+++ b/tests/snippets/rego/test_rules.txt
@@ -0,0 +1,749 @@
+---input---
+package example
+
+# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/
+
+### Constants
+
+a := {1, 2, 3}
+b := {4, 5, 6}
+c := a | b
+
+### Conditionals (Boolean)
+
+# p is true if ...
+p := true { 1 == 1 }
+
+# OR
+# with `import rego.v1` or `import future.keywords.if`
+p if { 2 > 1 }
+
+# OR
+p { 3 != 4 }
+
+
+### Conditionals
+
+# with `import rego.v1` or `import future.keywords.if`
+default a := 1
+a := 5   if { input.value > 3 }
+a := 100 if { input.value > 10 }
+
+
+### Incremental
+
+# a_set will contain values of x and values of y
+a_set[x] { x := [1, 2, 3][_] }
+a_set[y] { y := [4, 5, 6][_] }
+
+# alternatively, with `import rego.v1`, or `import future.keywords.contains` and `import future.keywords.if`
+a_set contains x if { x in [1, 2, 3] }
+a_set contains y if { y in [4, 5, 6] }
+
+# a_map will contain key->value pairs x->y and w->z
+a_map[x] := y if { x := "key1"; y := "value1" }
+a_map[w] := z if { w := "key2"; z := "value2" }
+
+
+### Functions (boolean)
+
+# with `import rego.v1` or `import future.keywords.if`
+f(x, y) if {
+    x < y
+}
+
+# OR
+
+f(x, y) := true if {
+    x + y > 10
+}
+
+### Functions (conditionals)
+
+# with `import rego.v1` or `import future.keywords.if`
+f(x) := "A" if { x >= 90 }
+f(x) := "B" if { x >= 80; x < 90 }
+f(x) := "C" if { x >= 70; x < 80 }
+
+
+### Reference Heads
+
+# with `import rego.v1`, or `import future.keywords.contains` and `import future.keywords.if`
+fruit.apple.seeds = 12 if input == "apple"             # complete document (single value rule)
+
+fruit.pineapple.colors contains x if x := "yellow"     # multi-value rule
+
+fruit.banana.phone[x] = "bananular" if x := "cellular" # single value rule
+fruit.banana.phone.cellular = "bananular" if true      # equivalent single value rule
+
+fruit.orange.color(x) = true if x == "orange"          # function
+
+---tokens---
+'package'     Keyword
+' '           Text
+'example'     Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# Rego Policy References: https://www.openpolicyagent.org/docs/latest/policy-reference/' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Constants' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'a'           Name
+' '           Text
+':='          Operator
+' '           Text
+'{'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'b'           Name
+' '           Text
+':='          Operator
+' '           Text
+'{'           Punctuation
+'4'           Literal.Number
+','           Punctuation
+' '           Text
+'5'           Literal.Number
+','           Punctuation
+' '           Text
+'6'           Literal.Number
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'c'           Name
+' '           Text
+':='          Operator
+' '           Text
+'a'           Name
+' '           Text
+'|'           Operator
+' '           Text
+'b'           Name
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Conditionals (Boolean)' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# p is true if ...' Comment.Single
+'\n'          Text.Whitespace
+
+'p'           Name
+' '           Text
+':='          Operator
+' '           Text
+'true'        Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'1'           Literal.Number
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# OR'        Comment.Single
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'p'           Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'2'           Literal.Number
+' '           Text
+'>'           Operator
+' '           Text
+'1'           Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# OR'        Comment.Single
+'\n'          Text.Whitespace
+
+'p'           Name
+' '           Text
+'{'           Punctuation
+' '           Text
+'3'           Literal.Number
+' '           Text
+'!='          Operator
+' '           Text
+'4'           Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Conditionals' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'default'     Keyword
+' '           Text
+'a'           Name
+' '           Text
+':='          Operator
+' '           Text
+'1'           Literal.Number
+'\n'          Text.Whitespace
+
+'a'           Name
+' '           Text
+':='          Operator
+' '           Text
+'5'           Literal.Number
+'   '         Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'input'       Name.Builtin
+'.'           Punctuation
+'value'       Name
+' '           Text
+'>'           Operator
+' '           Text
+'3'           Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'a'           Name
+' '           Text
+':='          Operator
+' '           Text
+'100'         Literal.Number
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'input'       Name.Builtin
+'.'           Punctuation
+'value'       Name
+' '           Text
+'>'           Operator
+' '           Text
+'10'          Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Incremental' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# a_set will contain values of x and values of y' Comment.Single
+'\n'          Text.Whitespace
+
+'a_set'       Name
+'['           Punctuation
+'x'           Name
+']'           Punctuation
+' '           Text
+'{'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+':='          Operator
+' '           Text
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+'['           Punctuation
+'_'           Name
+']'           Punctuation
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'a_set'       Name
+'['           Punctuation
+'y'           Name
+']'           Punctuation
+' '           Text
+'{'           Punctuation
+' '           Text
+'y'           Name
+' '           Text
+':='          Operator
+' '           Text
+'['           Punctuation
+'4'           Literal.Number
+','           Punctuation
+' '           Text
+'5'           Literal.Number
+','           Punctuation
+' '           Text
+'6'           Literal.Number
+']'           Punctuation
+'['           Punctuation
+'_'           Name
+']'           Punctuation
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# alternatively, with `import rego.v1`, or `import future.keywords.contains` and `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'a_set'       Name
+' '           Text
+'contains'    Keyword
+' '           Text
+'x'           Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+'in'          Keyword
+' '           Text
+'['           Punctuation
+'1'           Literal.Number
+','           Punctuation
+' '           Text
+'2'           Literal.Number
+','           Punctuation
+' '           Text
+'3'           Literal.Number
+']'           Punctuation
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'a_set'       Name
+' '           Text
+'contains'    Keyword
+' '           Text
+'y'           Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'y'           Name
+' '           Text
+'in'          Keyword
+' '           Text
+'['           Punctuation
+'4'           Literal.Number
+','           Punctuation
+' '           Text
+'5'           Literal.Number
+','           Punctuation
+' '           Text
+'6'           Literal.Number
+']'           Punctuation
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# a_map will contain key->value pairs x->y and w->z' Comment.Single
+'\n'          Text.Whitespace
+
+'a_map'       Name
+'['           Punctuation
+'x'           Name
+']'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'y'           Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"key1"'      Literal.String.Double
+';'           Punctuation
+' '           Text
+'y'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"value1"'    Literal.String.Double
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'a_map'       Name
+'['           Punctuation
+'w'           Name
+']'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'z'           Name
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'w'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"key2"'      Literal.String.Double
+';'           Punctuation
+' '           Text
+'z'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"value2"'    Literal.String.Double
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Functions (boolean)' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'f'           Name
+'('           Punctuation
+'x'           Name
+','           Punctuation
+' '           Text
+'y'           Name
+')'           Punctuation
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'x'           Name
+' '           Text
+'<'           Operator
+' '           Text
+'y'           Name
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# OR'        Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'f'           Name
+'('           Punctuation
+'x'           Name
+','           Punctuation
+' '           Text
+'y'           Name
+')'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'true'        Keyword
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text
+'x'           Name
+' '           Text
+'+'           Operator
+' '           Text
+'y'           Name
+' '           Text
+'>'           Operator
+' '           Text
+'10'          Literal.Number
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Functions (conditionals)' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1` or `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'"A"'         Literal.String.Double
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+'>='          Operator
+' '           Text
+'90'          Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'"B"'         Literal.String.Double
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+'>='          Operator
+' '           Text
+'80'          Literal.Number
+';'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+'<'           Operator
+' '           Text
+'90'          Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'f'           Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+' '           Text
+':='          Operator
+' '           Text
+'"C"'         Literal.String.Double
+' '           Text
+'if'          Keyword
+' '           Text
+'{'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+'>='          Operator
+' '           Text
+'70'          Literal.Number
+';'           Punctuation
+' '           Text
+'x'           Name
+' '           Text
+'<'           Operator
+' '           Text
+'80'          Literal.Number
+' '           Text
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'### Reference Heads' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'# with `import rego.v1`, or `import future.keywords.contains` and `import future.keywords.if`' Comment.Single
+'\n'          Text.Whitespace
+
+'fruit'       Name
+'.'           Punctuation
+'apple'       Name
+'.'           Punctuation
+'seeds'       Name
+' '           Text
+'='           Operator
+' '           Text
+'12'          Literal.Number
+' '           Text
+'if'          Keyword
+' '           Text
+'input'       Name.Builtin
+' '           Text
+'=='          Operator
+' '           Text
+'"apple"'     Literal.String.Double
+'             ' Text
+'# complete document (single value rule)' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'fruit'       Name
+'.'           Punctuation
+'pineapple'   Name
+'.'           Punctuation
+'colors'      Name
+' '           Text
+'contains'    Keyword
+' '           Text
+'x'           Name
+' '           Text
+'if'          Keyword
+' '           Text
+'x'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"yellow"'    Literal.String.Double
+'     '       Text
+'# multi-value rule' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'fruit'       Name
+'.'           Punctuation
+'banana'      Name
+'.'           Punctuation
+'phone'       Name
+'['           Punctuation
+'x'           Name
+']'           Punctuation
+' '           Text
+'='           Operator
+' '           Text
+'"bananular"' Literal.String.Double
+' '           Text
+'if'          Keyword
+' '           Text
+'x'           Name
+' '           Text
+':='          Operator
+' '           Text
+'"cellular"'  Literal.String.Double
+' '           Text
+'# single value rule' Comment.Single
+'\n'          Text.Whitespace
+
+'fruit'       Name
+'.'           Punctuation
+'banana'      Name
+'.'           Punctuation
+'phone'       Name
+'.'           Punctuation
+'cellular'    Name
+' '           Text
+'='           Operator
+' '           Text
+'"bananular"' Literal.String.Double
+' '           Text
+'if'          Keyword
+' '           Text
+'true'        Keyword
+'      '      Text
+'# equivalent single value rule' Comment.Single
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'fruit'       Name
+'.'           Punctuation
+'orange'      Name
+'.'           Punctuation
+'color'       Name
+'('           Punctuation
+'x'           Name
+')'           Punctuation
+' '           Text
+'='           Operator
+' '           Text
+'true'        Keyword
+' '           Text
+'if'          Keyword
+' '           Text
+'x'           Name
+' '           Text
+'=='          Operator
+' '           Text
+'"orange"'    Literal.String.Double
+'          '  Text
+'# function'  Comment.Single
+'\n'          Text.Whitespace
diff --git a/tests/test_ruby.py b/tests/test_ruby.py
index bfa06eb6..57faf9c2 100644
--- a/tests/test_ruby.py
+++ b/tests/test_ruby.py
@@ -7,8 +7,7 @@
 """
 
 import pytest
-
-from pygments.token import Name
+from pygments.token import Name, Token
 from pygments.lexers.ruby import RubyLexer
 
 
@@ -51,3 +50,14 @@ def test_negative_method_names(lexer, method_name):
 
     text = 'def ' + method_name
     assert list(lexer.get_tokens(text))[-2] != (Name.Function, method_name)
+
+
+def testShiftLeftWithoutSpace(lexer):
+    fragment = u'[]<<a\n'
+    tokens = [
+        (Token.Operator, u'['),
+        (Token.Operator, u']'),
+        (Token.Operator, u'<<'),
+        (Token.Name, u'a'),
+        (Token.Text, u'\n'),
+    ]
diff --git a/tests/test_terminal_formatter.py b/tests/test_terminal_formatter.py
index 20ae9a90..3489c852 100644
--- a/tests/test_terminal_formatter.py
+++ b/tests/test_terminal_formatter.py
@@ -61,6 +61,7 @@ class MyStyle(Style):
         Token.String:     'ansibrightblue bg:ansired',
         Token.Number:     'ansibrightgreen bg:ansigreen',
         Token.Number.Hex: 'ansigreen bg:ansibrightred',
+        Token.Other:      '#3366AA'
     }
 
 
@@ -76,7 +77,8 @@ async def function(a,b,c, *d, **kwarg:Bool)->Bool:
 
 def test_style_html():
     style = HtmlFormatter(style=MyStyle).get_style_defs()
-    assert '#555555' in style, "ansigray for comment not html css style"
+    assert '#555' in style, "ansigray for comment not html css style"
+    assert '36A' in style, "6-character style was not shortened"
 
 
 def test_others_work():