Repository: gristlabs/asttokens

Issue:
Compatibility with Python 3.12 and unreleased astroid 3.0.0
Hello,

I'd like to ask for help, in Fedora, we are rebuilding all Python packages with the upcoming Python 3.12.0. I've tested astroid from the main branch (last commit https://github.com/pylint-dev/astroid/commit/8d57ce2f3e226c2ac3cdd7f6a57dac2dd5ec5a4b at this time) and it works with 3.12, but asttokens fails with following failures. Thank you.

~~~~
=================================== FAILURES ===================================
___________________ TestAstroid.test_adjacent_joined_strings ___________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a8c00ad0>
data = "_\n(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = "_\n(f')", type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = "_\n(f')\n", type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = "_\n(f')\n", filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           (f')
E            ^
E       SyntaxError: unterminated f-string literal (detected at line 2)

/usr/lib64/python3.12/ast.py:52: SyntaxError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_adjacent_joined_strings>

        def test_adjacent_joined_strings(self):
            source = """
    foo = f'x y z' \\
    f'''a b c''' f"u v w"
    bar = ('x y z'   # comment2
           'a b c'   # comment3
           f'u v w'
          )
    """
>           m = self.create_mark_checker(source)

tests/test_mark_tokens.py:327: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:801: in parse_snippet
    return self.module.parse('_\n(' + text + ')').body[1].value
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a8c00ad0>
data = "_\n(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           unterminated f-string literal (detected at line 2) (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
__________________________ TestAstroid.test_fixture9 ___________________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a80692e0>
data = '_\nclass Aaaa(base):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = '_\nclass Aaaa(base):', type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = '_\nclass Aaaa(base):\n', type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = '_\nclass Aaaa(base):\n', filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           class Aaaa(base):
E                            ^
E       IndentationError: expected an indented block after class definition on line 2

/usr/lib64/python3.12/ast.py:52: IndentationError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_fixture9>

>   def test_fixture9(self): self.verify_fixture_file('astroid/module2.py')

tests/test_mark_tokens.py:181: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:140: in verify_fixture_file
    tested_nodes = m.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:804: in parse_snippet
    return self.module.parse('_\n' + text).body[1]
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a80692e0>
data = '_\nclass Aaaa(base):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           expected an indented block after class definition on line 2 (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
__________________________ TestAstroid.test_fstrings ___________________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a806bbf0>
data = '_\n((f")', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = '_\n((f")', type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = '_\n((f")\n', type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = '_\n((f")\n', filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           ((f")
E             ^
E       SyntaxError: unterminated f-string literal (detected at line 2)

/usr/lib64/python3.12/ast.py:52: SyntaxError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_fstrings>

    def test_fstrings(self):
      for source in (
        '(f"He said his name is {name!r}.",)',
        "f'{function(kwarg=24)}'",
        'a = f"""result: {value:{width}.{precision}}"""',
        """[f"abc {a['x']} def"]""",
        "def t():\n  return f'{function(kwarg=24)}'"):
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:316: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:801: in parse_snippet
    return self.module.parse('_\n(' + text + ')').body[1].value
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a806bbf0>
data = '_\n((f")', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           unterminated f-string literal (detected at line 2) (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
_________________________ TestAstroid.test_sys_modules _________________________

self = <astroid.builder.AstroidBuilder object at 0x7f28a750fcb0>
data = "_\n(_DeadlockError(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = "_\n(_DeadlockError(f')", type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = "_\n(_DeadlockError(f')\n", type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = "_\n(_DeadlockError(f')\n", filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           (_DeadlockError(f')
E                           ^
E       SyntaxError: unterminated f-string literal (detected at line 2)

/usr/lib64/python3.12/ast.py:52: SyntaxError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_sys_modules>

    def test_sys_modules(self):
      """
      Verify all nodes on source files obtained from sys.modules.
      This can take a long time as there are many modules,
      so it only tests all modules if the environment variable
      ASTTOKENS_SLOW_TESTS has been set.
      """
      modules = list(sys.modules.values())
      if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
        modules = modules[:20]
    
      start = time()
      for module in modules:
        # Don't let this test (which runs twice) take longer than 13 minutes
        # to avoid the travis build time limit of 30 minutes
        if time() - start > 13 * 60:
          break
    
        try:
          filename = inspect.getsourcefile(module)
        except TypeError:
          continue
    
        if not filename:
          continue
    
        filename = os.path.abspath(filename)
        print(filename)
        try:
          with io.open(filename) as f:
            source = f.read()
        except OSError:
          continue
    
        if self.is_astroid_test and (
            # Astroid fails with a syntax error if a type comment is on its own line
            re.search(r'^\s*# type: ', source, re.MULTILINE)
            # Astroid can fail on this file, specifically raising an exception at this line of code:
            #     lambda node: node.name == "NamedTuple" and node.parent.name == "typing"
            # with the error:
            #     AttributeError: 'If' object has no attribute 'name'
            # See https://github.com/gristlabs/asttokens/runs/7602147792
            # I think the code that causes the problem is:
            #     if sys.version_info >= (3, 11):
            #         NamedTuple = typing.NamedTuple
            or filename.endswith("typing_extensions.py")
        ):
          print('Skipping', filename)
          continue
    
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:673: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:801: in parse_snippet
    return self.module.parse('_\n(' + text + ')').body[1].value
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f28a750fcb0>
data = "_\n(_DeadlockError(f')", modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           unterminated f-string literal (detected at line 2) (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
----------------------------- Captured stdout call -----------------------------
/usr/lib64/python3.12/importlib/_bootstrap.py
_________________ TestMarkTokens.test_adjacent_joined_strings __________________

self = <tests.test_mark_tokens.TestMarkTokens testMethod=test_adjacent_joined_strings>

        def test_adjacent_joined_strings(self):
            source = """
    foo = f'x y z' \\
    f'''a b c''' f"u v w"
    bar = ('x y z'   # comment2
           'a b c'   # comment3
           f'u v w'
          )
    """
>           m = self.create_mark_checker(source)

tests/test_mark_tokens.py:327: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:84: in verify_all_nodes
    self.check_get_text_tokenless(node, test_case, text)
tests/tools.py:142: in check_get_text_tokenless
    test_case.assertEqual(text, text_tokenless, ast.dump(node))
E   AssertionError: "f'" != 'f\'x y z\' \\\nf\'\'\'a b c\'\'\' f"u v w"'
E   - f'
E   + f'x y z' \
E   + f'''a b c''' f"u v w"
E    : JoinedStr(values=[Constant(value='x y za b cu v w')])
_________________________ TestMarkTokens.test_fstrings _________________________

self = <tests.test_mark_tokens.TestMarkTokens testMethod=test_fstrings>

    def test_fstrings(self):
      for source in (
        '(f"He said his name is {name!r}.",)',
        "f'{function(kwarg=24)}'",
        'a = f"""result: {value:{width}.{precision}}"""',
        """[f"abc {a['x']} def"]""",
        "def t():\n  return f'{function(kwarg=24)}'"):
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:316: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:84: in verify_all_nodes
    self.check_get_text_tokenless(node, test_case, text)
tests/tools.py:142: in check_get_text_tokenless
    test_case.assertEqual(text, text_tokenless, ast.dump(node))
E   AssertionError: '(f"' != '(f"He said his name is {name!r}.",)'
E   - (f"
E   + (f"He said his name is {name!r}.",)
E    : Tuple(elts=[JoinedStr(values=[Constant(value='He said his name is '), FormattedValue(value=Name(id='name', ctx=Load()), conversion=114), Constant(value='.')])], ctx=Load())
_______________________ TestMarkTokens.test_sys_modules ________________________

self = <tests.test_mark_tokens.TestMarkTokens testMethod=test_sys_modules>

    def test_sys_modules(self):
      """
      Verify all nodes on source files obtained from sys.modules.
      This can take a long time as there are many modules,
      so it only tests all modules if the environment variable
      ASTTOKENS_SLOW_TESTS has been set.
      """
      modules = list(sys.modules.values())
      if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
        modules = modules[:20]
    
      start = time()
      for module in modules:
        # Don't let this test (which runs twice) take longer than 13 minutes
        # to avoid the travis build time limit of 30 minutes
        if time() - start > 13 * 60:
          break
    
        try:
          filename = inspect.getsourcefile(module)
        except TypeError:
          continue
    
        if not filename:
          continue
    
        filename = os.path.abspath(filename)
        print(filename)
        try:
          with io.open(filename) as f:
            source = f.read()
        except OSError:
          continue
    
        if self.is_astroid_test and (
            # Astroid fails with a syntax error if a type comment is on its own line
            re.search(r'^\s*# type: ', source, re.MULTILINE)
            # Astroid can fail on this file, specifically raising an exception at this line of code:
            #     lambda node: node.name == "NamedTuple" and node.parent.name == "typing"
            # with the error:
            #     AttributeError: 'If' object has no attribute 'name'
            # See https://github.com/gristlabs/asttokens/runs/7602147792
            # I think the code that causes the problem is:
            #     if sys.version_info >= (3, 11):
            #         NamedTuple = typing.NamedTuple
            or filename.endswith("typing_extensions.py")
        ):
          print('Skipping', filename)
          continue
    
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:673: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:84: in verify_all_nodes
    self.check_get_text_tokenless(node, test_case, text)
tests/tools.py:142: in check_get_text_tokenless
    test_case.assertEqual(text, text_tokenless, ast.dump(node))
E   AssertionError: "_DeadlockError(f'" != "_DeadlockError(f'deadlock detected by {self!r}')"
E   - _DeadlockError(f'
E   + _DeadlockError(f'deadlock detected by {self!r}')
E    : Call(func=Name(id='_DeadlockError', ctx=Load()), args=[JoinedStr(values=[Constant(value='deadlock detected by '), FormattedValue(value=Name(id='self', ctx=Load()), conversion=114)])], keywords=[])
----------------------------- Captured stdout call -----------------------------
/usr/lib64/python3.12/importlib/_bootstrap.py
____________________ TestTokenless.test_get_text_tokenless _____________________

self = <tests.test_tokenless.TestTokenless testMethod=test_get_text_tokenless>

    def test_get_text_tokenless(self):
      atok = ASTText(source)
    
      for node in ast.walk(atok.tree):
        if not isinstance(node, (ast.arguments, ast.arg)):
>         self.check_node(atok, node)

tests/test_tokenless.py:70: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_tokenless.py:108: in check_node
    self.assertEqual(atok_text, ast_text, node)
E   AssertionError: '' != '{xx + 22}'
E   + {xx + 22}
E    : <ast.FormattedValue object at 0x7f28a95a4ed0>
_____________________________ test_combine_tokens ______________________________

    def test_combine_tokens():
      from tokenize import TokenInfo, generate_tokens, ERRORTOKEN, OP, NUMBER, NAME
      from asttokens.util import combine_tokens, patched_generate_tokens
    
      text = "â„˜Â·2=1"
      original_tokens = list(generate_tokens(io.StringIO(text).readline))[:4]
>     assert original_tokens == [
        TokenInfo(ERRORTOKEN, string='â„˜', start=(1, 0), end=(1, 1), line='â„˜Â·2=1'),
        TokenInfo(ERRORTOKEN, string='Â·', start=(1, 1), end=(1, 2), line='â„˜Â·2=1'),
        TokenInfo(NUMBER, string='2', start=(1, 2), end=(1, 3), line='â„˜Â·2=1'),
        TokenInfo(OP, string='=', start=(1, 3), end=(1, 4), line='â„˜Â·2=1'),
      ]
E     AssertionError: assert [TokenInfo(ty...line='â„˜Â·2=1')] == [TokenInfo(ty...line='â„˜Â·2=1')]
E       At index 0 diff: TokenInfo(type=1 (NAME), string='â„˜Â·2', start=(1, 0), end=(1, 3), line='â„˜Â·2=1') != TokenInfo(type=66 (ERRORTOKEN), string='â„˜', start=(1, 0), end=(1, 1), line='â„˜Â·2=1')
E       Full diff:
E         [
E       -  TokenInfo(type=66 (ERRORTOKEN), string='â„˜', start=(1, 0), end=(1, 1), line='â„˜Â·2=1'),
E       -  TokenInfo(type=66 (ERRORTOKEN), string='Â·', start=(1, 1), end=(1, 2), line='â„˜Â·2=1'),
E       -  TokenInfo(type=2 (NUMBER), string='2', start=(1, 2), end=(1, 3), line='â„˜Â·2=1'),
E       ?                 ^   ^ - -                         ^...
E       
E       ...Full output truncated (6 lines hidden), use '-vv' to show

tests/test_util.py:126: AssertionError
=========================== short test summary info ============================
FAILED tests/test_astroid.py::TestAstroid::test_adjacent_joined_strings - ast...
FAILED tests/test_astroid.py::TestAstroid::test_fixture9 - astroid.exceptions...
FAILED tests/test_astroid.py::TestAstroid::test_fstrings - astroid.exceptions...
FAILED tests/test_astroid.py::TestAstroid::test_sys_modules - astroid.excepti...
FAILED tests/test_mark_tokens.py::TestMarkTokens::test_adjacent_joined_strings
FAILED tests/test_mark_tokens.py::TestMarkTokens::test_fstrings - AssertionEr...
FAILED tests/test_mark_tokens.py::TestMarkTokens::test_sys_modules - Assertio...
FAILED tests/test_tokenless.py::TestTokenless::test_get_text_tokenless - Asse...
FAILED tests/test_util.py::test_combine_tokens - AssertionError: assert [Toke...
============ 9 failed, 106 passed, 1 skipped, 25 warnings in 4.10s =============
~~~~

Discussion:
alexmojaki: GHA no longer supports Python 2.7: https://github.com/actions/setup-python/issues/672

While there are ways we could continue testing 2.7, this seems like a good time to let it go. Anyone using 2.7 can still use an older version of asttokens (i.e. the current one) and they should be fine. Newer versions of asttokens have generally been motivated by newer versions of Python anyway, as with this PR.
hroncok: Using astroid from git HEAD and this PR, we get this in Fedora:

```
=================================== FAILURES ===================================
__________________________ TestAstroid.test_fixture9 ___________________________

self = <astroid.builder.AstroidBuilder object at 0x7f4ba936e720>
data = '_\nclass Aaaa(base):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = '_\nclass Aaaa(base):', type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = '_\nclass Aaaa(base):\n', type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = '_\nclass Aaaa(base):\n', filename = '<unknown>', mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           class Aaaa(base):
E                            ^
E       IndentationError: expected an indented block after class definition on line 2

/usr/lib64/python3.12/ast.py:52: IndentationError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_fixture9>

>   def test_fixture9(self): self.verify_fixture_file('astroid/module2.py')

tests/test_mark_tokens.py:181: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:140: in verify_fixture_file
    tested_nodes = m.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:804: in parse_snippet
    return self.module.parse('_\n' + text).body[1]
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f4ba936e720>
data = '_\nclass Aaaa(base):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           expected an indented block after class definition on line 2 (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
_________________________ TestAstroid.test_sys_modules _________________________

self = <astroid.builder.AstroidBuilder object at 0x7f4ba81a0b90>
data = '_\ndef create_module(self, spec):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
>           node, parser_module = _parse_string(data, type_comments=True)

/usr/lib/python3.12/site-packages/astroid/builder.py:176: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data = '_\ndef create_module(self, spec):', type_comments = True

    def _parse_string(
        data: str, type_comments: bool = True
    ) -> tuple[ast.Module, ParserModule]:
        parser_module = get_parser_module(type_comments=type_comments)
        try:
>           parsed = parser_module.parse(data + "\n", type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/builder.py:482: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = ParserModule(unary_op_classes={<class 'ast.UAdd'>: '+', <class 'ast.USub'>: '-', <class 'ast.Not'>: 'not', <class 'ast...<class 'ast.Store'>: <Context.Store: 2>, <class 'ast.Del'>: <Context.Del: 3>, <class 'ast.Param'>: <Context.Store: 2>})
string = '_\ndef create_module(self, spec):\n', type_comments = True

    def parse(self, string: str, type_comments: bool = True) -> ast.Module:
>       return ast.parse(string, type_comments=type_comments)

/usr/lib/python3.12/site-packages/astroid/_ast.py:26: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

source = '_\ndef create_module(self, spec):\n', filename = '<unknown>'
mode = 'exec'

    def parse(source, filename='<unknown>', mode='exec', *,
              type_comments=False, feature_version=None):
        """
        Parse the source into an AST node.
        Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
        Pass type_comments=True to get back type comments where the syntax allows.
        """
        flags = PyCF_ONLY_AST
        if type_comments:
            flags |= PyCF_TYPE_COMMENTS
        if feature_version is None:
            feature_version = -1
        elif isinstance(feature_version, tuple):
            major, minor = feature_version  # Should be a 2-tuple.
            if major != 3:
                raise ValueError(f"Unsupported major version: {major}")
            feature_version = minor
        # Else it should be an int giving the minor version for 3.x.
>       return compile(source, filename, mode, flags,
                       _feature_version=feature_version)
E         File "<unknown>", line 2
E           def create_module(self, spec):
E                                         ^
E       IndentationError: expected an indented block after function definition on line 2

/usr/lib64/python3.12/ast.py:52: IndentationError

The above exception was the direct cause of the following exception:

self = <tests.test_astroid.TestAstroid testMethod=test_sys_modules>

    def test_sys_modules(self):
      """
      Verify all nodes on source files obtained from sys.modules.
      This can take a long time as there are many modules,
      so it only tests all modules if the environment variable
      ASTTOKENS_SLOW_TESTS has been set.
      """
      modules = list(sys.modules.values())
      if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
        modules = modules[:20]
    
      start = time()
      for module in modules:
        # Don't let this test (which runs twice) take longer than 13 minutes
        # to avoid the travis build time limit of 30 minutes
        if time() - start > 13 * 60:
          break
    
        try:
          filename = inspect.getsourcefile(module)
        except TypeError:
          continue
    
        if not filename:
          continue
    
        filename = os.path.abspath(filename)
        print(filename)
        try:
          with io.open(filename) as f:
            source = f.read()
        except OSError:
          continue
    
        if self.is_astroid_test and (
            # Astroid fails with a syntax error if a type comment is on its own line
            re.search(r'^\s*# type: ', source, re.MULTILINE)
            # Astroid can fail on this file, specifically raising an exception at this line of code:
            #     lambda node: node.name == "NamedTuple" and node.parent.name == "typing"
            # with the error:
            #     AttributeError: 'If' object has no attribute 'name'
            # See https://github.com/gristlabs/asttokens/runs/7602147792
            # I think the code that causes the problem is:
            #     if sys.version_info >= (3, 11):
            #         NamedTuple = typing.NamedTuple
            or filename.endswith("typing_extensions.py")
        ):
          print('Skipping', filename)
          continue
    
>       self.create_mark_checker(source)

tests/test_mark_tokens.py:673: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_mark_tokens.py:49: in create_mark_checker
    checker.verify_all_nodes(self)
tests/tools.py:107: in verify_all_nodes
    rebuilt_node = test_case.parse_snippet(text, node)
tests/test_mark_tokens.py:804: in parse_snippet
    return self.module.parse('_\n' + text).body[1]
/usr/lib/python3.12/site-packages/astroid/builder.py:305: in parse
    return builder.string_build(code, modname=module_name, path=path)
/usr/lib/python3.12/site-packages/astroid/builder.py:146: in string_build
    module, builder = self._data_build(data, modname, path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <astroid.builder.AstroidBuilder object at 0x7f4ba81a0b90>
data = '_\ndef create_module(self, spec):', modname = '', path = None

    def _data_build(
        self, data: str, modname: str, path: str | None
    ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
        """Build tree node from data and add some informations."""
        try:
            node, parser_module = _parse_string(data, type_comments=True)
        except (TypeError, ValueError, SyntaxError) as exc:
>           raise AstroidSyntaxError(
                "Parsing Python code failed:\n{error}",
                source=data,
                modname=modname,
                path=path,
                error=exc,
            ) from exc
E           astroid.exceptions.AstroidSyntaxError: Parsing Python code failed:
E           expected an indented block after function definition on line 2 (<unknown>, line 2)

/usr/lib/python3.12/site-packages/astroid/builder.py:178: AstroidSyntaxError
----------------------------- Captured stdout call -----------------------------
/usr/lib64/python3.12/importlib/_bootstrap.py
/usr/lib64/python3.12/importlib/_bootstrap_external.py
=========================== short test summary info ============================
FAILED tests/test_astroid.py::TestAstroid::test_fixture9 - astroid.exceptions...
FAILED tests/test_astroid.py::TestAstroid::test_sys_modules - astroid.excepti...
```
hroncok: Is it possible that this would in fact be a problem in astroid?
aqeelat: @alexmojaki This is not a review of the fix you created but of the deprecation of Python 2.7.

Don't forget to remove the python 2 and 2.7 classifiers and the dependency on six.
alexmojaki: > Don't forget to remove the python 2 and 2.7 classifiers

Thanks, this is an important reminder.

> and the dependency on six. Although, I think that should be done in a separate PR that gets merged before this one.

I don't really see why continuing to depend on six is a critical problem that should block this PR (although this PR isn't actually ready to merge yet), even without Python 2 support. 

> I can give it a shot if you create an GH issue and assign it to me.

I think we need to decide to actually drop Python 2 support, and that's up to @dsagal. But I've opened https://github.com/gristlabs/asttokens/issues/111 for that discussion to continue there.

Patch:
diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py
index 3afce53..32b42d7 100644
--- a/asttokens/mark_tokens.py
+++ b/asttokens/mark_tokens.py
@@ -361,7 +361,34 @@ class MarkTokens(object):
                       last_token,  # type: util.Token
                       ):
     # type: (...) -> Tuple[util.Token, util.Token]
-    return self.handle_str(first_token, last_token)
+    if sys.version_info < (3, 12):
+      # Older versions don't tokenize the contents of f-strings
+      return self.handle_str(first_token, last_token)
+
+    last = first_token
+    while True:
+      if util.match_token(last, getattr(token, "FSTRING_START")):
+        # Python 3.12+ has tokens for the start (e.g. `f"`) and end (`"`)
+        # of the f-string. We can't just look for the next FSTRING_END
+        # because f-strings can be nested, e.g. f"{f'{x}'}", so we need
+        # to treat this like matching balanced parentheses.
+        count = 1
+        while count > 0:
+          last = self._code.next_token(last)
+          # mypy complains about token.FSTRING_START and token.FSTRING_END.
+          if util.match_token(last, getattr(token, "FSTRING_START")):
+            count += 1
+          elif util.match_token(last, getattr(token, "FSTRING_END")):
+            count -= 1
+        last_token = last
+        last = self._code.next_token(last_token)
+      elif util.match_token(last, token.STRING):
+        # Similar to handle_str, we also need to handle adjacent strings.
+        last_token = last
+        last = self._code.next_token(last_token)
+      else:
+        break
+    return (first_token, last_token)
 
   def visit_bytes(self, node, first_token, last_token):
     # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
diff --git a/asttokens/util.py b/asttokens/util.py
index 1db9ffd..ad3be9a 100644
--- a/asttokens/util.py
+++ b/asttokens/util.py
@@ -446,6 +446,10 @@ if sys.version_info[:2] >= (3, 8):
     Add a special attribute `_broken_positions` to nodes inside f-strings
     if the lineno/col_offset cannot be trusted.
     """
+    if sys.version_info >= (3, 12):
+      # f-strings were weirdly implemented until https://peps.python.org/pep-0701/
+      # In Python 3.12, inner nodes have sensible positions.
+      return
     for joinedstr in walk(tree):
       if not isinstance(joinedstr, ast.JoinedStr):
         continue
@@ -457,6 +461,11 @@ if sys.version_info[:2] >= (3, 8):
           if not fstring_positions_work():
             for child in walk(part.value):
               setattr(child, '_broken_positions', True)
+              if isinstance(child, ast.JoinedStr):
+                # Recursively handle this inner JoinedStr in the same way.
+                # While this is usually automatic for other nodes,
+                # the children of f-strings are explicitly excluded in iter_children_ast.
+                annotate_fstring_nodes(child)
 
           if part.format_spec:  # this is another JoinedStr
             # Again, the standard positions span the full f-string.
diff --git a/setup.cfg b/setup.cfg
index df73b0e..588a105 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -29,6 +29,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
 


Test patch:
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f31ac16..9d452a8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -20,6 +20,7 @@ jobs:
           - 3.9
           - '3.10'
           - 3.11
+          - 3.12.0-rc.1
           # As per https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#pypy list of versions
           - pypy-2.7
           - pypy-3.6
diff --git a/tests/test_astroid.py b/tests/test_astroid.py
index a5cc6d7..13b88ba 100644
--- a/tests/test_astroid.py
+++ b/tests/test_astroid.py
@@ -35,5 +35,12 @@ class TestAstroid(test_mark_tokens.TestMarkTokens):
   @staticmethod
   def create_asttokens(source):
     builder = astroid.builder.AstroidBuilder()
-    tree = builder.string_build(source)
+    try:
+      tree = builder.string_build(source)
+    except AttributeError as e:
+      raise AstroidTreeException(str(e))
     return ASTTokens(source, tree=tree)
+
+
+class AstroidTreeException(Exception):
+  pass
diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py
index bc94563..d8a1f9c 100644
--- a/tests/test_mark_tokens.py
+++ b/tests/test_mark_tokens.py
@@ -627,6 +627,8 @@ j  # not a complex number, just a name
       so it only tests all modules if the environment variable
       ASTTOKENS_SLOW_TESTS has been set.
       """
+      from .test_astroid import AstroidTreeException
+
       modules = list(sys.modules.values())
       if not os.environ.get('ASTTOKENS_SLOW_TESTS'):
         modules = modules[:20]
@@ -640,7 +642,7 @@ j  # not a complex number, just a name
 
         try:
           filename = inspect.getsourcefile(module)
-        except TypeError:
+        except Exception:  # some modules raise weird errors
           continue
 
         if not filename:
@@ -657,20 +659,21 @@ j  # not a complex number, just a name
         if self.is_astroid_test and (
             # Astroid fails with a syntax error if a type comment is on its own line
             re.search(r'^\s*# type: ', source, re.MULTILINE)
-            # Astroid can fail on this file, specifically raising an exception at this line of code:
-            #     lambda node: node.name == "NamedTuple" and node.parent.name == "typing"
-            # with the error:
-            #     AttributeError: 'If' object has no attribute 'name'
-            # See https://github.com/gristlabs/asttokens/runs/7602147792
-            # I think the code that causes the problem is:
-            #     if sys.version_info >= (3, 11):
-            #         NamedTuple = typing.NamedTuple
-            or filename.endswith("typing_extensions.py")
         ):
           print('Skipping', filename)
           continue
 
-        self.create_mark_checker(source)
+        try:
+          self.create_mark_checker(source)
+        except AstroidTreeException:
+          # Astroid sometimes fails with errors like:
+          #     AttributeError: 'TreeRebuilder' object has no attribute 'visit_typealias'
+          # See https://github.com/gristlabs/asttokens/actions/runs/6015907789/job/16318767911?pr=110
+          # Should be fixed in the next astroid release:
+          #     https://github.com/pylint-dev/pylint/issues/8782#issuecomment-1669967220
+          # Note that this exception is raised before asttokens is even involved,
+          # it's purely an astroid bug that we can safely ignore.
+          continue
 
   if six.PY3:
     def test_dict_merge(self):
diff --git a/tests/test_tokenless.py b/tests/test_tokenless.py
index f20c438..955089f 100644
--- a/tests/test_tokenless.py
+++ b/tests/test_tokenless.py
@@ -98,7 +98,7 @@ class TestTokenless(unittest.TestCase):
       ast_text = ast.get_source_segment(source, node, padded=padded)
       atok_text = atok.get_text(node, padded=padded)
       if ast_text:
-        if (
+        if sys.version_info < (3, 12) and (
           ast_text.startswith("f") and isinstance(node, (ast.Str, ast.FormattedValue))
           or is_fstring_format_spec(node)
           or (not fstring_positions_work() and is_fstring_internal_node(node))
@@ -120,6 +120,28 @@ class TestTokenless(unittest.TestCase):
     with self.assertRaises(NotImplementedError):
       ASTText(source, tree)
 
+  def test_nested_fstrings(self):
+    f1 = 'f"a {1+2} b {3+4} c"'
+    f2 = "f'd {" + f1 + "} e'"
+    f3 = "f'''{" + f2 + "}{" + f1 + "}'''"
+    f4 = 'f"""{' + f3 + '}"""'
+    s = 'f = ' + f4
+    atok = ASTText(s)
+    self.assertEqual(atok.get_text(atok.tree), s)
+    n4 = atok.tree.body[0].value
+    n3 = n4.values[0].value
+    n2 = n3.values[0].value
+    n1 = n2.values[1].value
+    self.assertEqual(atok.get_text(n4), f4)
+    if fstring_positions_work():
+      self.assertEqual(atok.get_text(n3), f3)
+      self.assertEqual(atok.get_text(n2), f2)
+      self.assertEqual(atok.get_text(n1), f1)
+    else:
+      self.assertEqual(atok.get_text(n3), '')
+      self.assertEqual(atok.get_text(n2), '')
+      self.assertEqual(atok.get_text(n1), '')
+
 
 class TestFstringPositionsWork(unittest.TestCase):
   def test_fstring_positions_work(self):
diff --git a/tests/test_util.py b/tests/test_util.py
index 8f8bad5..4e4f395 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals, print_function
 
 import ast
 import io
+import sys
 import token
 import unittest
 
@@ -122,27 +123,38 @@ if six.PY3:
     from asttokens.util import combine_tokens, patched_generate_tokens
 
     text = "℘·2=1"
-    original_tokens = list(generate_tokens(io.StringIO(text).readline))[:4]
-    assert original_tokens == [
-      TokenInfo(ERRORTOKEN, string='℘', start=(1, 0), end=(1, 1), line='℘·2=1'),
-      TokenInfo(ERRORTOKEN, string='·', start=(1, 1), end=(1, 2), line='℘·2=1'),
-      TokenInfo(NUMBER, string='2', start=(1, 2), end=(1, 3), line='℘·2=1'),
-      TokenInfo(OP, string='=', start=(1, 3), end=(1, 4), line='℘·2=1'),
-    ]
-    assert combine_tokens(original_tokens[:1]) == [
-      TokenInfo(NAME, string='℘', start=(1, 0), end=(1, 1), line='℘·2=1'),
-    ]
-    assert combine_tokens(original_tokens[:2]) == [
-      TokenInfo(NAME, string='℘·', start=(1, 0), end=(1, 2), line='℘·2=1'),
-    ]
-    assert combine_tokens(original_tokens[:3]) == [
-      TokenInfo(NAME, string='℘·2', start=(1, 0), end=(1, 3), line='℘·2=1'),
-    ]
+    original_tokens = []
+    for tok in generate_tokens(io.StringIO(text).readline):
+      original_tokens.append(tok)
+      if tok.type == OP:
+        break
 
-    assert list(patched_generate_tokens(iter(original_tokens))) == [
+    correct_tokens = [
       TokenInfo(NAME, string='℘·2', start=(1, 0), end=(1, 3), line='℘·2=1'),
       TokenInfo(OP, string='=', start=(1, 3), end=(1, 4), line='℘·2=1'),
     ]
+    if sys.version_info >= (3, 12):
+      # The tokenizing bug was fixed in 3.12, so the original tokens are correct,
+      # rather than starting with false ERRORTOKENs.
+      assert original_tokens == correct_tokens
+    else:
+      assert original_tokens == [
+        TokenInfo(ERRORTOKEN, string='℘', start=(1, 0), end=(1, 1), line='℘·2=1'),
+        TokenInfo(ERRORTOKEN, string='·', start=(1, 1), end=(1, 2), line='℘·2=1'),
+        TokenInfo(NUMBER, string='2', start=(1, 2), end=(1, 3), line='℘·2=1'),
+        TokenInfo(OP, string='=', start=(1, 3), end=(1, 4), line='℘·2=1'),
+      ]
+      assert combine_tokens(original_tokens[:1]) == [
+        TokenInfo(NAME, string='℘', start=(1, 0), end=(1, 1), line='℘·2=1'),
+      ]
+      assert combine_tokens(original_tokens[:2]) == [
+        TokenInfo(NAME, string='℘·', start=(1, 0), end=(1, 2), line='℘·2=1'),
+      ]
+      assert combine_tokens(original_tokens[:3]) == [
+        TokenInfo(NAME, string='℘·2', start=(1, 0), end=(1, 3), line='℘·2=1'),
+      ]
+
+    assert list(patched_generate_tokens(iter(original_tokens))) == correct_tokens
     assert list(patched_generate_tokens(iter(original_tokens[:-1]))) == [
       TokenInfo(NAME, string='℘·2', start=(1, 0), end=(1, 3), line='℘·2=1'),
     ]