// File: /workspace/github_cpp_makefile_star2_O2/rofl0r[P]ascc/tokenizer.c
// Function: tokenizer_next_real
int tokenizer_next_real(struct tokenizer *t, struct token* out) {
	char *s = t->buf;
	out->value = 0;
	int c = 0;
	if(t->peeking) {
		*out = t->peek_token;
		t->peeking = 0;
		return 1;
	}
	while(1) {
		c = tokenizer_getc(t);
		if(c == EOF) break;

		/* components of multi-line comment marker might be terminals themselves */
		if(sequence_follows(t, c, t->marker[MT_MULTILINE_COMMENT_START])) {
			ignore_until(t, t->marker[MT_MULTILINE_COMMENT_END], strlen(t->marker[MT_MULTILINE_COMMENT_START]));
			continue;
		}
		if(sequence_follows(t, c, t->marker[MT_SINGLELINE_COMMENT_START])) {
			ignore_until(t, "\n", strlen(t->marker[MT_SINGLELINE_COMMENT_START]));
			continue;
		}
		if(is_sep(c)) {
			if(s != t->buf && c == '\\' && !isspace(s[-1])) {
				c = tokenizer_getc(t);
				if(c == '\n') continue;
				tokenizer_ungetc(t, c);
				c = '\\';
			} else if(is_plus_or_minus(c) && s > t->buf+1 &&
				  (s[-1] == 'E' || s[-1] == 'e') && is_valid_float_until(t->buf, s-1)) {
				goto process_char;
			} else if(c == '.' && s != t->buf && is_valid_float_until(t->buf, s) == 1) {
				goto process_char;
			} else if(c == '.' && s == t->buf) {
				int jump = 0;
				c = tokenizer_getc(t);
				if(isdigit(c)) jump = 1;
				tokenizer_ungetc(t, c);
				c = '.';
				if(jump) goto process_char;
			}
			tokenizer_ungetc(t, c);
			break;
		}
		if((t->flags & TF_PARSE_WIDE_STRINGS) && s == t->buf && c == 'L') {
			c = tokenizer_getc(t);
			tokenizer_ungetc(t, c);
			tokenizer_ungetc(t, 'L');
			if(c == '\'' || c == '\"') break;
		}

process_char:;
		s = assign_bufchar(t, s, c);
		if(t->column + 1 >= MAX_TOK_LEN) {
			out->type = TT_OVERFLOW;
			return apply_coords(t, out, s, 0);
		}
	}
	if(s == t->buf) {
		if(c == EOF) {
			out->type = TT_EOF;
			return apply_coords(t, out, s, 1);
		}

		int wide = 0;
		c = tokenizer_getc(t);
		if((t->flags & TF_PARSE_WIDE_STRINGS) && c == 'L') {
			c = tokenizer_getc(t);
			assert(c == '\'' || c == '\"');
			wide = 1;
			goto string_handling;
		} else if (c == '.' && sequence_follows(t, c, "...")) {
			strcpy(t->buf, "...");
			out->type = TT_ELLIPSIS;
			return apply_coords(t, out, s+3, 1);
		}

		{
			int i;
			for(i = 0; i < t->custom_count; i++)
				if(sequence_follows(t, c, t->custom_tokens[i])) {
					const char *p = t->custom_tokens[i];
					while(*p) {
						s = assign_bufchar(t, s, *p);
						p++;
					}
					*s = 0;
					out->type = TT_CUSTOM + i;
					return apply_coords(t, out, s, 1);
				}
		}

string_handling:
		s = assign_bufchar(t, s, c);
		*s = 0;
		//s = assign_bufchar(t, s, 0);
		if(c == '"' || c == '\'')
			if(t->flags & TF_PARSE_STRINGS) return get_string(t, c, out, wide);
		out->type = TT_SEP;
		out->value = c;
		if(c == '\n') {
			apply_coords(t, out, s, 1);
			t->line++;
			t->column=0;
			return 1;
		}
		return apply_coords(t, out, s, 1);
	}
	//s = assign_bufchar(t, s, 0);
	*s = 0;
	out->type = categorize(t->buf);
	return apply_coords(t, out, s, out->type != TT_UNKNOWN);
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x38, %rsp
movq %rsi, %r12
andl $0x0, 0xc(%rsi)
movq %rdi, %r14
cmpl $0x0, 0x18(%rdi)
je 0x161ac
movups 0x1178(%r14), %xmm0
movups %xmm0, (%r12)
andl $0x0, 0x18(%r14)
pushq $0x1
popq %rax
addq $0x38, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x120(%r14), %rax
movq %rax, 0x20(%rsp)
movl $0x120, %r13d           # imm = 0x120
leaq 0x5032(%rip), %rbp      # 0x1b1f7
leaq (%r14,%r13), %rax
movq %rax, 0x18(%rsp)
leaq (%r14,%r13), %rbx
decq %rbx
movq %r14, %rdi
callq 0x15dfb
cmpl $-0x1, %eax
je 0x163e9
movl %eax, %r15d
movq 0x1160(%r14), %rdx
movq %r14, %rdi
movl %eax, %esi
callq 0x1609a
testl %eax, %eax
je 0x16219
movq 0x1160(%r14), %rdi
movq 0x1168(%r14), %r15
callq 0x3190
movq %r14, %rdi
movq %r15, %rsi
jmp 0x16241
movq 0x1158(%r14), %rdx
movq %r14, %rdi
movl %r15d, %esi
callq 0x1609a
testl %eax, %eax
je 0x1624a
movq 0x1158(%r14), %rdi
callq 0x3190
movq %r14, %rdi
movq %rbp, %rsi
movl %eax, %edx
callq 0x16111
jmp 0x161d5
testb %r15b, %r15b
js 0x16297
movslq %r15d, %rax
leaq 0x5367(%rip), %rcx      # 0x1b5c0
cmpb $0x0, (%rax,%rcx)
je 0x16297
cmpq $0x120, %r13            # imm = 0x120
je 0x162a6
cmpl $0x5c, %r15d
jne 0x162a6
callq 0x3340
movq (%rax), %rax
movsbq (%rbx), %rcx
testb $0x20, 0x1(%rax,%rcx,2)
jne 0x162a6
movq %r14, %rdi
callq 0x15dfb
cmpl $0xa, %eax
je 0x161d5
jmp 0x16411
testb $0x2, 0x10(%r14)
jne 0x162ef
movl %r15d, %ebx
jmp 0x163af
leal -0x2b(%r15), %eax
testl $0xfffffffd, %eax       # imm = 0xFFFFFFFD
sete %al
cmpq $0x122, %r13            # imm = 0x122
setae %cl
andb %al, %cl
cmpb $0x1, %cl
jne 0x1633d
movzbl (%rbx), %eax
orl $0x20, %eax
cmpl $0x65, %eax
jne 0x1641f
movq 0x20(%rsp), %rdi
movq %rbx, %rsi
callq 0x16a61
movl %r15d, %ebx
testl %eax, %eax
jne 0x163af
jmp 0x1641f
cmpq $0x120, %r13            # imm = 0x120
setne %al
cmpl $0x4c, %r15d
setne %cl
orb %al, %cl
movl %r15d, %ebx
jne 0x163af
movq %r14, %rdi
callq 0x15dfb
movl %eax, %ebx
movq %r14, %rdi
movl %eax, %esi
callq 0x15e51
movq %r14, %rdi
pushq $0x4c
popq %rsi
callq 0x15e51
cmpl $0x22, %ebx
je 0x16433
cmpl $0x27, %ebx
jne 0x163af
jmp 0x16433
cmpq $0x120, %r13            # imm = 0x120
sete %al
cmpl $0x2e, %r15d
setne %cl
orb %al, %cl
jne 0x1636e
movq 0x20(%rsp), %rdi
movq 0x18(%rsp), %rsi
callq 0x16a61
pushq $0x2e
popq %rbx
cmpl $0x1, %eax
je 0x163af
jmp 0x169fb
cmpl $0x2e, %r15d
jne 0x1641f
cmpq $0x120, %r13            # imm = 0x120
jne 0x1641f
movq %r14, %rdi
callq 0x15dfb
movl %eax, %ebx
callq 0x3340
movq (%rax), %rax
movslq %ebx, %rsi
movzwl (%rax,%rsi,2), %ebx
movq %r14, %rdi
callq 0x15e51
btl $0xb, %ebx
pushq $0x2e
popq %rbx
jae 0x1641f
incl 0xc(%r14)
movb %bl, (%r14,%r13)
incq %r13
movl 0xc(%r14), %eax
movl $0xfffff001, %ecx       # imm = 0xFFFFF001
addl %ecx, %eax
cmpl $0xfffff000, %eax       # imm = 0xFFFFF000
jae 0x161c5
addq %r14, %r13
movl $0xb, (%r12)
movq %r14, %rdi
movq %r12, %rsi
movq %r13, %rdx
jmp 0x169f4
cmpq $0x120, %r13            # imm = 0x120
jne 0x164a2
movl $0xe, (%r12)
pushq $0x1
popq %rcx
movq %r14, %rdi
movq %r12, %rsi
movq 0x18(%rsp), %rdx
jmp 0x16773
movq %r14, %rdi
movl %eax, %esi
callq 0x15e51
pushq $0x5c
popq %r15
movq %r14, %rdi
movl %r15d, %esi
callq 0x15e51
cmpq $0x120, %r13            # imm = 0x120
jne 0x164a2
movq %r14, %rdi
callq 0x15dfb
movl %eax, %ebp
testb $0x2, 0x10(%r14)
sete %al
cmpl $0x4c, %ebp
setne %r13b
orb %al, %r13b
je 0x16786
cmpl $0x2e, %ebp
jne 0x1684d
leaq 0x45fc(%rip), %rdx      # 0x1aa61
pushq $0x2e
popq %rsi
movq %r14, %rdi
callq 0x1609a
testl %eax, %eax
je 0x1684d
movl $0x2e2e2e, 0x120(%r14) # imm = 0x2E2E2E
movl $0x4, (%r12)
movq 0x18(%rsp), %rdx
addq $0x3, %rdx
pushq $0x1
popq %rcx
movq %r14, %rdi
movq %r12, %rsi
jmp 0x16773
movq 0x18(%rsp), %rax
movb $0x0, (%rax)
movq 0x20(%rsp), %r13
movsbl (%r13), %eax
xorl %ecx, %ecx
movq %rax, 0x30(%rsp)
cmpl $0x2d, %eax
sete %cl
movq %rcx, 0x28(%rsp)
movb (%r13,%rcx), %al
movb %al, 0xf(%rsp)
cmpb $0x30, %al
jne 0x16579
movq 0x28(%rsp), %rax
movzbl 0x1(%r13,%rax), %eax
orl $0x20, %eax
cmpl $0x78, %eax
jne 0x16579
movq %r12, 0x10(%rsp)
movq 0x28(%rsp), %rax
leaq (%rax,%r14), %r12
addq $0x122, %r12            # imm = 0x122
xorl %ebp, %ebp
pushq $0x5
popq %rax
movl %eax, 0x8(%rsp)
leaq 0x5088(%rip), %rbx      # 0x1b59b
pushq $0x11
popq %r15
movsbq (%r12,%rbp), %r13
testq %r13, %r13
je 0x16754
callq 0x30b0
movq (%rax), %rax
movl (%rax,%r13,4), %esi
movq %rbx, %rdi
movq %r15, %rdx
callq 0x3380
testq %rax, %rax
je 0x16546
incq %rbp
jmp 0x16517
testq %rbp, %rbp
movq 0x10(%rsp), %r12
movq 0x20(%rsp), %r13
pushq $0x1
popq %r15
je 0x16579
movq 0x28(%rsp), %rax
addq %r14, %rax
leaq (%rax,%rbp), %rdi
addq $0x122, %rdi            # imm = 0x122
callq 0x16be0
testl %eax, %eax
jne 0x1675d
movq 0x30(%rsp), %rax
addl $-0x2b, %eax
xorl %ebp, %ebp
testb $-0x3, %al
setne 0x20(%rsp)
sete %bpl
movb (%r13,%rbp), %r15b
pushq $0x7
popq %rax
movl %eax, 0x8(%rsp)
cmpb $0x30, %r15b
jne 0x165be
movsbq 0x1(%r13,%rbp), %rbx
testq %rbx, %rbx
je 0x16759
callq 0x3340
movq (%rax), %rax
testb $0x8, 0x1(%rax,%rbx,2)
jne 0x1661c
leaq (%r14,%rbp), %rbx
addq $0x120, %rbx            # imm = 0x120
testb %r15b, %r15b
je 0x16759
callq 0x3340
movq (%rax), %rax
movsbq %r15b, %rcx
testb $0x8, 0x1(%rax,%rcx,2)
je 0x165f1
incq %rbp
movb 0x1(%rbx), %r15b
incq %rbx
jmp 0x165c9
testq %rbp, %rbp
setne %al
cmpq $0x2, %rbp
setae %cl
movb 0x20(%rsp), %dl
orb %cl, %dl
testb %dl, %al
pushq $0x1
popq %r15
je 0x1661c
movq %rbx, %rdi
callq 0x16be0
testl %eax, %eax
jne 0x1675d
movq %r12, 0x10(%rsp)
cmpb $0x30, 0xf(%rsp)
jne 0x1666c
movq 0x28(%rsp), %rax
leaq (%rax,%r14), %r12
addq $0x121, %r12            # imm = 0x121
movb $0x30, %al
pushq $0x6
popq %rbp
leaq 0x4f6f(%rip), %rbx      # 0x1b5b3
pushq $0x9
popq %r15
testb %al, %al
je 0x16750
movsbl %al, %esi
movq %rbx, %rdi
movq %r15, %rdx
callq 0x3380
testq %rax, %rax
je 0x1666c
movb (%r12), %al
incq %r12
jmp 0x16648
movq 0x30(%rsp), %rax
addl $-0x2b, %eax
andb $-0x3, %al
cmpb $0x1, %al
adcq $0x0, %r13
movl $0x0, 0x20(%rsp)
xorl %r15d, %r15d
xorl %ebx, %ebx
movsbq (%r13), %r12
testq %r12, %r12
je 0x16738
callq 0x30b0
cmpb $0x2e, %r12b
jne 0x166b2
testl %ebx, %ebx
pushq $0x1
popq %rbx
je 0x16730
jmp 0x167ea
movq %r13, %rbp
movq (%rax), %rax
movl (%rax,%r12,4), %r13d
cmpl $0x66, %r13d
je 0x167c6
callq 0x3340
movq (%rax), %rax
testb $0x8, 0x1(%rax,%r12,2)
jne 0x16700
cmpl $0x65, %r13d
jne 0x167ea
testl %r15d, %r15d
je 0x167ea
movb 0x1(%rbp), %dl
addb $-0x2b, %dl
xorl %ecx, %ecx
testb $-0x3, %dl
setne %sil
jne 0x16709
leaq 0x2(%rbp), %rdx
jmp 0x1670d
pushq $0x1
popq %r15
movq %rbp, %r13
jmp 0x16730
leaq 0x1(%rbp), %rdx
movb %sil, %cl
negq %rcx
movsbq 0x2(%rbp,%rcx), %rcx
testb $0x8, 0x1(%rax,%rcx,2)
pushq $0x1
popq %r15
movl %r15d, 0x20(%rsp)
movq %rdx, %r13
je 0x167ea
incq %r13
jmp 0x16689
testl %r15d, %r15d
je 0x167ea
orl %ebx, 0x20(%rsp)
jne 0x167dd
jmp 0x167ea
movl %ebp, 0x8(%rsp)
movq 0x10(%rsp), %r12
pushq $0x1
popq %r15
movl 0x8(%rsp), %eax
movl %eax, (%r12)
movq %r14, %rdi
movq %r12, %rsi
movq 0x18(%rsp), %rdx
movl %r15d, %ecx
addq $0x38, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
jmp 0x16abb
movq %r14, %rdi
callq 0x15dfb
movl %eax, %ebp
cmpl $0x22, %eax
je 0x16879
cmpl $0x27, %ebp
movq 0x18(%rsp), %r15
je 0x1687e
leaq 0x4ce3(%rip), %rdi      # 0x1b491
leaq 0x4bba(%rip), %rsi      # 0x1b36f
leaq 0x4cec(%rip), %rcx      # 0x1b4a8
movl $0x1de, %edx            # imm = 0x1DE
callq 0x3240
cmpb $0x0, 0x1(%rbp)
jne 0x167ea
orl 0x20(%rsp), %ebx
setne %al
testl %r15d, %r15d
setne %cl
testb %cl, %al
je 0x167ea
pushq $0x8
popq %rax
movl %eax, 0x8(%rsp)
pushq $0x1
popq %r15
jmp 0x16843
pushq $0xa
popq %rdi
xorl %r15d, %r15d
movq 0x30(%rsp), %rax
testb %al, %al
movl %edi, 0x8(%rsp)
js 0x16843
movl %eax, %ecx
leaq 0x4e3a(%rip), %rax      # 0x1b640
cmpb $0x1, (%rcx,%rax)
movq 0x10(%rsp), %r12
jne 0x1675d
leaq 0x121(%r14), %rdx
pushq $0x1
popq %rcx
movsbl (%rdx), %esi
testl %esi, %esi
je 0x169be
testb %sil, %sil
js 0x1675d
movl %esi, %esi
incq %rdx
cmpb $0x0, (%rsi,%rax)
jne 0x1681f
jmp 0x1675d
movq 0x10(%rsp), %r12
jmp 0x1675d
xorl %ebx, %ebx
movq 0x18(%rsp), %r15
movslq 0x14(%r14), %rax
cmpq %rax, %rbx
jge 0x1687e
movq 0x20(%r14,%rbx,8), %rdx
movq %r14, %rdi
movl %ebp, %esi
callq 0x1609a
testl %eax, %eax
jne 0x16985
incq %rbx
jmp 0x16854
movq 0x18(%rsp), %r15
incl 0xc(%r14)
movb %bpl, (%r15)
movb $0x0, 0x1(%r15)
incq %r15
cmpl $0x27, %ebp
je 0x1689b
cmpl $0x22, %ebp
jne 0x16954
testb $0x1, 0x10(%r14)
jne 0x168b4
movl $0x9, (%r12)
movl %ebp, 0xc(%r12)
jmp 0x169ad
movq %r12, 0x10(%rsp)
leaq 0x121(%r14), %r15
movq 0x1120(%r14), %rax
movq 0x20(%rsp), %rcx
leaq (%rcx,%rax), %r12
addq $-0x2, %r12
xorl %eax, %eax
testl %eax, %eax
sete 0x18(%rsp)
cmpq %r12, %r15
jae 0x169ca
movq %r14, %rdi
callq 0x15dfb
movl %eax, %ebx
cmpl $0x5c, %eax
jne 0x16904
movq %r14, %rdi
callq 0x15dfb
cmpl $0xa, %eax
je 0x168dd
jmp 0x16925
cmpl $0xa, %ebx
jne 0x1691a
testb $0x1, 0x18(%rsp)
movb $0x1, 0x18(%rsp)
je 0x168dd
jmp 0x16a06
cmpl $-0x1, %ebx
je 0x169df
jmp 0x1692f
movq %r14, %rdi
movl %eax, %esi
callq 0x15e51
xorl %eax, %eax
testb $0x1, 0x18(%rsp)
je 0x16948
cmpl %ebp, %ebx
je 0x16a29
xorl %eax, %eax
cmpl $0x5c, %ebx
sete %al
incl 0xc(%r14)
movb %bl, (%r15)
incq %r15
jmp 0x168d6
movl $0x9, (%r12)
movl %ebp, 0xc(%r12)
cmpl $0xa, %ebp
jne 0x169ad
pushq $0x1
popq %rcx
movq %r14, %rdi
movq %r12, %rsi
movq %r15, %rdx
callq 0x16abb
incl 0x8(%r14)
andl $0x0, 0xc(%r14)
jmp 0x1619a
movq 0x20(%r14,%rbx,8), %rax
movb (%rax), %cl
testb %cl, %cl
je 0x1699f
incl 0xc(%r14)
movb %cl, (%r15)
incq %r15
incq %rax
jmp 0x1698a
movb $0x0, (%r15)
addl $0x3e8, %ebx            # imm = 0x3E8
movl %ebx, (%r12)
pushq $0x1
popq %rcx
movq %r14, %rdi
movq %r12, %rsi
movq %r15, %rdx
jmp 0x16773
movl %ecx, %r15d
movl %ecx, 0x8(%rsp)
jmp 0x1675d
movb $0x0, 0x111f(%r14)
movq 0x10(%rsp), %rsi
movl $0xb, (%rsi)
jmp 0x169ee
movq 0x10(%rsp), %rsi
movl $0xe, (%rsi)
movb $0x0, (%r15)
movq %r14, %rdi
movq %r15, %rdx
xorl %ecx, %ecx
jmp 0x16773
pushq $0x2e
popq %rax
movl %eax, %r15d
jmp 0x1641f
pushq $0xa
popq %rsi
movq %r14, %rdi
callq 0x15e51
movq 0x10(%rsp), %rsi
movl $0xa, (%rsi)
incl 0xc(%r14)
movb $0x0, (%r15)
incq %r15
jmp 0x169ee
incl 0xc(%r14)
movb %bpl, (%r15)
movb $0x0, 0x1(%r15)
incq %r15
xorl %eax, %eax
cmpb $0x22, %bpl
sete %al
testb %r13b, %r13b
pushq $0x2
popq %rcx
pushq $0xc
popq %rdx
cmovnel %ecx, %edx
orl %eax, %edx
movq 0x10(%rsp), %rsi
movl %edx, (%rsi)
pushq $0x1
popq %rcx
movq %r14, %rdi
jmp 0x169b6