Lib/site-packages/Cython/Compiler/Tests/TestScanning.py

from __future__ import unicode_literals

import unittest
from io import StringIO
import string

from .. import Scanning
from ..Symtab import ModuleScope
from ..TreeFragment import StringParseContext
from ..Errors import init_thread

# generate some fake code - just a bunch of lines of the form "a0 a1 ..."
code = []
for ch in string.ascii_lowercase:
    line = " ".join(["%s%s" % (ch, n) for n in range(10)])
    code.append(line)
code = "\n".join(code)

init_thread()


class TestScanning(unittest.TestCase):
    def make_scanner(self):
        source = Scanning.StringSourceDescriptor("fake code", code)
        buf = StringIO(code)
        context = StringParseContext("fake context")
        scope = ModuleScope("fake_module", None, None)

        return Scanning.PyrexScanner(buf, source, scope=scope, context=context)

    def test_put_back_positions(self):
        scanner = self.make_scanner()

        self.assertEqual(scanner.sy, "IDENT")
        self.assertEqual(scanner.systring, "a0")
        scanner.next()
        self.assertEqual(scanner.sy, "IDENT")
        self.assertEqual(scanner.systring, "a1")
        a1pos = scanner.position()
        self.assertEqual(a1pos[1:], (1, 3))
        a2peek = scanner.peek()  # shouldn't mess up the position
        self.assertEqual(a1pos, scanner.position())
        scanner.next()
        self.assertEqual(a2peek, (scanner.sy, scanner.systring))

        # find next line
        while scanner.sy != "NEWLINE":
            scanner.next()

        line_sy = []
        line_systring = []
        line_pos = []

        scanner.next()
        while scanner.sy != "NEWLINE":
            line_sy.append(scanner.sy)
            line_systring.append(scanner.systring)
            line_pos.append(scanner.position())
            scanner.next()

        for sy, systring, pos in zip(
            line_sy[::-1], line_systring[::-1], line_pos[::-1]
        ):
            scanner.put_back(sy, systring, pos)

        n = 0
        while scanner.sy != "NEWLINE":
            self.assertEqual(scanner.sy, line_sy[n])
            self.assertEqual(scanner.systring, line_systring[n])
            self.assertEqual(scanner.position(), line_pos[n])
            scanner.next()
            n += 1

        self.assertEqual(n, len(line_pos))

    def test_tentatively_scan(self):
        scanner = self.make_scanner()
        with Scanning.tentatively_scan(scanner) as errors:
            while scanner.sy != "NEWLINE":
                scanner.next()
        self.assertFalse(errors)

        scanner.next()
        self.assertEqual(scanner.systring, "b0")
        pos = scanner.position()
        with Scanning.tentatively_scan(scanner) as errors:
            while scanner.sy != "NEWLINE":
                scanner.next()
                if scanner.systring == "b7":
                    scanner.error("Oh no not b7!")
                    break
        self.assertTrue(errors)
        self.assertEqual(scanner.systring, "b0")  # state has been restored
        self.assertEqual(scanner.position(), pos)
        scanner.next()
        self.assertEqual(scanner.systring, "b1")  # and we can keep going again
        scanner.next()
        self.assertEqual(scanner.systring, "b2")  # and we can keep going again

        with Scanning.tentatively_scan(scanner) as error:
            scanner.error("Something has gone wrong with the current symbol")
        self.assertEqual(scanner.systring, "b2")
        scanner.next()
        self.assertEqual(scanner.systring, "b3")

        # test a few combinations of nested scanning
        sy1, systring1 = scanner.sy, scanner.systring
        pos1 = scanner.position()
        with Scanning.tentatively_scan(scanner):
            scanner.next()
            sy2, systring2 = scanner.sy, scanner.systring
            pos2 = scanner.position()
            with Scanning.tentatively_scan(scanner):
                with Scanning.tentatively_scan(scanner):
                    scanner.next()
                    scanner.next()
                    scanner.error("Ooops")
                self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
            self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
            scanner.error("eee")
        self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
        with Scanning.tentatively_scan(scanner):
            scanner.next()
            scanner.next()
            with Scanning.tentatively_scan(scanner):
                scanner.next()
                # no error - but this block should be unwound by the outer block too
            scanner.next()
            scanner.error("Oooops")
        self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))




if __name__ == "__main__":
    unittest.main()