# Buggy code
def lex(self, subdir):
    line_start = 0
    lineno = 1
    loc = 0
    par_count = 0
    bracket_count = 0
    col = 0
    newline_rx = re.compile(r"(?<!\\)((?:\\\\)*)\\n")
    while loc < len(self.code):
        matched = False
        value = None
        for tid, reg in self.token_specification:
            mo = reg.match(self.code, loc)
            if mo:
                curline = lineno
                curline_start = line_start
                col = mo.start() - line_start
                matched = True
                span_start = loc
                loc = mo.end()
                span_end = loc
                bytespan = (span_start, span_end)
                match_text = mo.group()
                if tid == "ignore" or tid == "comment":
                    break
                elif tid == "lparen":
                    par_count += 1
                elif tid == "rparen":
                    par_count -= 1
                elif tid == "lbracket":
                    bracket_count += 1
                elif tid == "rbracket":
                    bracket_count -= 1
                elif tid == "dblquote":
                    raise ParseException(
                        "Double quotes are not supported. Use single quotes.",
                        self.getline(line_start),
                        lineno,
                        col,
                    )
                elif tid == "string":
                    # Handle here and not on the regexp to give a better error message.
                    if match_text.find("\n") != -1:
                        mlog.warning(
                            """Newline character in a string detected, use ''' (three single quotes) for multiline strings instead.
This will become a hard error in a future Meson release.""",
                            self.getline(line_start),
                            lineno,
                            col,
                        )
                    value = match_text[1:-1].replace(r"\'", "'")
                    value = newline_rx.sub(r"\1\n", value)
                    value = value.replace(r" \\ ".strip(), r" \ ".strip())
                elif tid == "multiline_string":
                    tid = "string"
                    value = match_text[3:-3]
                    lines = match_text.split("\n")
                    if len(lines) > 1:
                        lineno += len(lines) - 1
                        line_start = mo.end() - len(lines[-1])
                elif tid == "number":
                    value = int(match_text)
                elif tid == "hexnumber":
                    tid = "number"
                    value = int(match_text, base=16)
                elif tid == "eol" or tid == "eol_cont":
                    lineno += 1
                    line_start = loc
                    if par_count > 0 or bracket_count > 0:
                        break
                elif tid == "id":
                    if match_text in self.keywords:
                        tid = match_text
                    else:
                        value = match_text
                yield Token(tid, subdir, curline_start, curline, col, bytespan, value)
                break
        if not matched:
            raise ParseException("lexer", self.getline(line_start), lineno, col)


# Traceback (ValueError)
tansell@tansell:~/github/mesonbuild/meson/test cases/common/100 print null$ meson build .
The Meson build system
Version: 0.45.0.dev1
Source dir: XXXX/github/mesonbuild/meson/test cases/common/100 print null
Build dir: XXXX/github/mesonbuild/meson/test cases/common/100 print null/build
Build type: native build
Project name: 100 print null
Build machine cpu family: x86_64
Build machine cpu: x86_64
Program find found: YES (/usr/bin/find)
Traceback (most recent call last):
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/mesonmain.py", line
363, in run
app.generate()
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/mesonmain.py", line
150, in generate
self._generate(env)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/mesonmain.py", line
197, in _generate
intr.run()
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreter.py", line 2992, in run
super().run()
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 173, in run
self.evaluate_codeblock(self.ast, start=1)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 195, in evaluate_codeblock
raise e
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 189, in evaluate_codeblock
self.evaluate_statement(cur)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 230, in evaluate_statement
return self.evaluate_foreach(cur)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 404, in evaluate_foreach
self.evaluate_codeblock(node.block)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 195, in evaluate_codeblock
raise e
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 189, in evaluate_codeblock
self.evaluate_statement(cur)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 232, in evaluate_statement
return self.evaluate_plusassign(cur)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 409, in evaluate_plusassign
addition = self.evaluate_statement(node.value)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 216, in evaluate_statement
return self.evaluate_arraystatement(cur)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 243, in evaluate_arraystatement
(arguments, kwargs) = self.reduce_arguments(cur.args)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 640, in reduce_arguments
reduced_pos = [self.evaluate_statement(arg) for arg in args.arguments]
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 640, in <listcomp>
reduced_pos = [self.evaluate_statement(arg) for arg in args.arguments]
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 200, in evaluate_statement
return self.function_call(cur)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 456, in function_call
return self.funcs[func_name](node, self.flatten(posargs), kwargs)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 55, in wrapped
return f(self, node, args, kwargs)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreterbase.py", line 47, in wrapped
return f(self, node, args, kwargs)
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreter.py", line 1614, in func_files
return [mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, fname) for fname in
args]
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/interpreter.py", line 1614, in <listcomp>
return [mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, fname) for fname in
args]
File "XXXX/.local/lib/python3.5/site-packages/mesonbuild/mesonlib.py", line 230, in from_source_file
if not os.path.isfile(os.path.join(source_root, subdir, fname)):
File "/usr/lib/python3.5/genericpath.py", line 30, in isfile
st = os.stat(path)
ValueError: embedded null byte

# Fixed code
def lex(self, subdir):
    line_start = 0
    lineno = 1
    loc = 0
    par_count = 0
    bracket_count = 0
    col = 0
    while loc < len(self.code):
        matched = False
        value = None
        for tid, reg in self.token_specification:
            mo = reg.match(self.code, loc)
            if mo:
                curline = lineno
                curline_start = line_start
                col = mo.start() - line_start
                matched = True
                span_start = loc
                loc = mo.end()
                span_end = loc
                bytespan = (span_start, span_end)
                match_text = mo.group()
                if tid == "ignore" or tid == "comment":
                    break
                elif tid == "lparen":
                    par_count += 1
                elif tid == "rparen":
                    par_count -= 1
                elif tid == "lbracket":
                    bracket_count += 1
                elif tid == "rbracket":
                    bracket_count -= 1
                elif tid == "dblquote":
                    raise ParseException(
                        "Double quotes are not supported. Use single quotes.",
                        self.getline(line_start),
                        lineno,
                        col,
                    )
                elif tid == "string":
                    # Handle here and not on the regexp to give a better error message.
                    if match_text.find("\n") != -1:
                        mlog.warning(
                            """Newline character in a string detected, use ''' (three single quotes) for multiline strings instead.
This will become a hard error in a future Meson release.""",
                            self.getline(line_start),
                            lineno,
                            col,
                        )
                    value = match_text[1:-1]
                    value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, value)
                elif tid == "multiline_string":
                    tid = "string"
                    value = match_text[3:-3]
                    value = ESCAPE_SEQUENCE_MULTI_RE.sub(decode_match, value)
                    lines = match_text.split("\n")
                    if len(lines) > 1:
                        lineno += len(lines) - 1
                        line_start = mo.end() - len(lines[-1])
                elif tid == "number":
                    value = int(match_text)
                elif tid == "hexnumber":
                    tid = "number"
                    value = int(match_text, base=16)
                elif tid == "eol" or tid == "eol_cont":
                    lineno += 1
                    line_start = loc
                    if par_count > 0 or bracket_count > 0:
                        break
                elif tid == "id":
                    if match_text in self.keywords:
                        tid = match_text
                    else:
                        value = match_text
                yield Token(tid, subdir, curline_start, curline, col, bytespan, value)
                break
        if not matched:
            raise ParseException("lexer", self.getline(line_start), lineno, col)