// File: rofl0r[P]ascc/tokenizer.c
// Function: tokenizer_next_real
int tokenizer_next_real(struct tokenizer *t, struct token* out) {
	char *s = t->buf;
	out->value = 0;
	int c = 0;
	if(t->peeking) {
		*out = t->peek_token;
		t->peeking = 0;
		return 1;
	}
	while(1) {
		c = tokenizer_getc(t);
		if(c == EOF) break;

		/* components of multi-line comment marker might be terminals themselves */
		if(sequence_follows(t, c, t->marker[MT_MULTILINE_COMMENT_START])) {
			ignore_until(t, t->marker[MT_MULTILINE_COMMENT_END], strlen(t->marker[MT_MULTILINE_COMMENT_START]));
			continue;
		}
		if(sequence_follows(t, c, t->marker[MT_SINGLELINE_COMMENT_START])) {
			ignore_until(t, "\n", strlen(t->marker[MT_SINGLELINE_COMMENT_START]));
			continue;
		}
		if(is_sep(c)) {
			if(s != t->buf && c == '\\' && !isspace(s[-1])) {
				c = tokenizer_getc(t);
				if(c == '\n') continue;
				tokenizer_ungetc(t, c);
				c = '\\';
			} else if(is_plus_or_minus(c) && s > t->buf+1 &&
				  (s[-1] == 'E' || s[-1] == 'e') && is_valid_float_until(t->buf, s-1)) {
				goto process_char;
			} else if(c == '.' && s != t->buf && is_valid_float_until(t->buf, s) == 1) {
				goto process_char;
			} else if(c == '.' && s == t->buf) {
				int jump = 0;
				c = tokenizer_getc(t);
				if(isdigit(c)) jump = 1;
				tokenizer_ungetc(t, c);
				c = '.';
				if(jump) goto process_char;
			}
			tokenizer_ungetc(t, c);
			break;
		}
		if((t->flags & TF_PARSE_WIDE_STRINGS) && s == t->buf && c == 'L') {
			c = tokenizer_getc(t);
			tokenizer_ungetc(t, c);
			tokenizer_ungetc(t, 'L');
			if(c == '\'' || c == '\"') break;
		}

process_char:;
		s = assign_bufchar(t, s, c);
		if(t->column + 1 >= MAX_TOK_LEN) {
			out->type = TT_OVERFLOW;
			return apply_coords(t, out, s, 0);
		}
	}
	if(s == t->buf) {
		if(c == EOF) {
			out->type = TT_EOF;
			return apply_coords(t, out, s, 1);
		}

		int wide = 0;
		c = tokenizer_getc(t);
		if((t->flags & TF_PARSE_WIDE_STRINGS) && c == 'L') {
			c = tokenizer_getc(t);
			assert(c == '\'' || c == '\"');
			wide = 1;
			goto string_handling;
		} else if (c == '.' && sequence_follows(t, c, "...")) {
			strcpy(t->buf, "...");
			out->type = TT_ELLIPSIS;
			return apply_coords(t, out, s+3, 1);
		}

		{
			int i;
			for(i = 0; i < t->custom_count; i++)
				if(sequence_follows(t, c, t->custom_tokens[i])) {
					const char *p = t->custom_tokens[i];
					while(*p) {
						s = assign_bufchar(t, s, *p);
						p++;
					}
					*s = 0;
					out->type = TT_CUSTOM + i;
					return apply_coords(t, out, s, 1);
				}
		}

string_handling:
		s = assign_bufchar(t, s, c);
		*s = 0;
		//s = assign_bufchar(t, s, 0);
		if(c == '"' || c == '\'')
			if(t->flags & TF_PARSE_STRINGS) return get_string(t, c, out, wide);
		out->type = TT_SEP;
		out->value = c;
		if(c == '\n') {
			apply_coords(t, out, s, 1);
			t->line++;
			t->column=0;
			return 1;
		}
		return apply_coords(t, out, s, 1);
	}
	//s = assign_bufchar(t, s, 0);
	*s = 0;
	out->type = categorize(t->buf);
	return apply_coords(t, out, s, out->type != TT_UNKNOWN);
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
subq $0x50, %rsp
movq %rdi, -0x10(%rbp)
movq %rsi, -0x18(%rbp)
movq -0x10(%rbp), %rax
addq $0x120, %rax            # imm = 0x120
movq %rax, -0x20(%rbp)
movq -0x18(%rbp), %rax
movl $0x0, 0xc(%rax)
movl $0x0, -0x24(%rbp)
movq -0x10(%rbp), %rax
cmpl $0x0, 0x18(%rax)
je 0x2e0ae
movq -0x18(%rbp), %rax
movq -0x10(%rbp), %rcx
movq 0x1178(%rcx), %rdx
movq %rdx, (%rax)
movq 0x1180(%rcx), %rcx
movq %rcx, 0x8(%rax)
movq -0x10(%rbp), %rax
movl $0x0, 0x18(%rax)
movl $0x1, -0x4(%rbp)
jmp 0x2e681
jmp 0x2e0b0
movq -0x10(%rbp), %rdi
callq 0x2d870
movl %eax, -0x24(%rbp)
cmpl $-0x1, -0x24(%rbp)
jne 0x2e0c7
jmp 0x2e3ac
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
movq -0x10(%rbp), %rax
movq 0x1160(%rax), %rdx
callq 0x2de40
cmpl $0x0, %eax
je 0x2e11b
movq -0x10(%rbp), %rax
movq %rax, -0x48(%rbp)
movq -0x10(%rbp), %rax
movq 0x1168(%rax), %rax
movq %rax, -0x40(%rbp)
movq -0x10(%rbp), %rax
movq 0x1160(%rax), %rdi
callq 0x41d0
movq -0x48(%rbp), %rdi
movq -0x40(%rbp), %rsi
movl %eax, %edx
callq 0x2df80
jmp 0x2e0b0
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
movq -0x10(%rbp), %rax
movq 0x1158(%rax), %rdx
callq 0x2de40
cmpl $0x0, %eax
je 0x2e166
movq -0x10(%rbp), %rax
movq %rax, -0x50(%rbp)
movq -0x10(%rbp), %rax
movq 0x1158(%rax), %rdi
callq 0x41d0
movq -0x50(%rbp), %rdi
movl %eax, %edx
leaq 0x600f(%rip), %rsi      # 0x3416b
callq 0x2df80
jmp 0x2e0b0
movl -0x24(%rbp), %edi
callq 0x2e690
cmpl $0x0, %eax
je 0x2e2fc
movq -0x20(%rbp), %rax
movq -0x10(%rbp), %rcx
addq $0x120, %rcx            # imm = 0x120
cmpq %rcx, %rax
je 0x2e1e1
cmpl $0x5c, -0x24(%rbp)
jne 0x2e1e1
callq 0x4400
movq (%rax), %rax
movq -0x20(%rbp), %rcx
movsbl -0x1(%rcx), %ecx
movslq %ecx, %rcx
movzwl (%rax,%rcx,2), %eax
andl $0x2000, %eax           # imm = 0x2000
cmpl $0x0, %eax
jne 0x2e1e1
movq -0x10(%rbp), %rdi
callq 0x2d870
movl %eax, -0x24(%rbp)
cmpl $0xa, -0x24(%rbp)
jne 0x2e1c9
jmp 0x2e0b0
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
callq 0x2d910
movl $0x5c, -0x24(%rbp)
jmp 0x2e2eb
movl -0x24(%rbp), %edi
callq 0x2e6d0
cmpl $0x0, %eax
je 0x2e242
movq -0x20(%rbp), %rax
movq -0x10(%rbp), %rcx
addq $0x120, %rcx            # imm = 0x120
addq $0x1, %rcx
cmpq %rcx, %rax
jbe 0x2e242
movq -0x20(%rbp), %rax
movsbl -0x1(%rax), %eax
cmpl $0x45, %eax
je 0x2e220
movq -0x20(%rbp), %rax
movsbl -0x1(%rax), %eax
cmpl $0x65, %eax
jne 0x2e242
movq -0x10(%rbp), %rdi
addq $0x120, %rdi            # imm = 0x120
movq -0x20(%rbp), %rsi
addq $-0x1, %rsi
callq 0x2e700
cmpl $0x0, %eax
je 0x2e242
jmp 0x2e35d
cmpl $0x2e, -0x24(%rbp)
jne 0x2e27a
movq -0x20(%rbp), %rax
movq -0x10(%rbp), %rcx
addq $0x120, %rcx            # imm = 0x120
cmpq %rcx, %rax
je 0x2e27a
movq -0x10(%rbp), %rdi
addq $0x120, %rdi            # imm = 0x120
movq -0x20(%rbp), %rsi
callq 0x2e700
cmpl $0x1, %eax
jne 0x2e27a
jmp 0x2e35d
cmpl $0x2e, -0x24(%rbp)
jne 0x2e2e5
movq -0x20(%rbp), %rax
movq -0x10(%rbp), %rcx
addq $0x120, %rcx            # imm = 0x120
cmpq %rcx, %rax
jne 0x2e2e5
movl $0x0, -0x28(%rbp)
movq -0x10(%rbp), %rdi
callq 0x2d870
movl %eax, -0x24(%rbp)
callq 0x4400
movq (%rax), %rax
movslq -0x24(%rbp), %rcx
movzwl (%rax,%rcx,2), %eax
andl $0x800, %eax            # imm = 0x800
cmpl $0x0, %eax
je 0x2e2c8
movl $0x1, -0x28(%rbp)
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
callq 0x2d910
movl $0x2e, -0x24(%rbp)
cmpl $0x0, -0x28(%rbp)
je 0x2e2e3
jmp 0x2e35d
jmp 0x2e2e5
jmp 0x2e2e7
jmp 0x2e2e9
jmp 0x2e2eb
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
callq 0x2d910
jmp 0x2e3ac
movq -0x10(%rbp), %rax
movl 0x10(%rax), %eax
andl $0x2, %eax
cmpl $0x0, %eax
je 0x2e35b
movq -0x20(%rbp), %rax
movq -0x10(%rbp), %rcx
addq $0x120, %rcx            # imm = 0x120
cmpq %rcx, %rax
jne 0x2e35b
cmpl $0x4c, -0x24(%rbp)
jne 0x2e35b
movq -0x10(%rbp), %rdi
callq 0x2d870
movl %eax, -0x24(%rbp)
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
callq 0x2d910
movq -0x10(%rbp), %rdi
movl $0x4c, %esi
callq 0x2d910
cmpl $0x27, -0x24(%rbp)
je 0x2e357
cmpl $0x22, -0x24(%rbp)
jne 0x2e359
jmp 0x2e3ac
jmp 0x2e35b
jmp 0x2e35d
movq -0x10(%rbp), %rdi
movq -0x20(%rbp), %rsi
movl -0x24(%rbp), %edx
callq 0x2df10
movq %rax, -0x20(%rbp)
movq -0x10(%rbp), %rax
movl 0xc(%rax), %eax
addl $0x1, %eax
cmpl $0x1000, %eax           # imm = 0x1000
jb 0x2e3a7
movq -0x18(%rbp), %rax
movl $0xb, (%rax)
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
xorl %ecx, %ecx
callq 0x2e7b0
movl %eax, -0x4(%rbp)
jmp 0x2e681
jmp 0x2e0b0
movq -0x20(%rbp), %rax
movq -0x10(%rbp), %rcx
addq $0x120, %rcx            # imm = 0x120
cmpq %rcx, %rax
jne 0x2e63f
cmpl $-0x1, -0x24(%rbp)
jne 0x2e3f2
movq -0x18(%rbp), %rax
movl $0xe, (%rax)
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
movl $0x1, %ecx
callq 0x2e7b0
movl %eax, -0x4(%rbp)
jmp 0x2e681
movl $0x0, -0x2c(%rbp)
movq -0x10(%rbp), %rdi
callq 0x2d870
movl %eax, -0x24(%rbp)
movq -0x10(%rbp), %rax
movl 0x10(%rax), %eax
andl $0x2, %eax
cmpl $0x0, %eax
je 0x2e45f
cmpl $0x4c, -0x24(%rbp)
jne 0x2e45f
movq -0x10(%rbp), %rdi
callq 0x2d870
movl %eax, -0x24(%rbp)
cmpl $0x27, -0x24(%rbp)
je 0x2e432
cmpl $0x22, -0x24(%rbp)
jne 0x2e434
jmp 0x2e453
leaq 0x611b(%rip), %rdi      # 0x34556
leaq 0x5ff2(%rip), %rsi      # 0x34434
movl $0x1de, %edx            # imm = 0x1DE
leaq 0x611f(%rip), %rcx      # 0x3456d
callq 0x4290
movl $0x1, -0x2c(%rbp)
jmp 0x2e57d
cmpl $0x2e, -0x24(%rbp)
jne 0x2e4c0
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
leaq 0x55a2(%rip), %rdx      # 0x33a15
callq 0x2de40
cmpl $0x0, %eax
je 0x2e4c0
movq -0x10(%rbp), %rdi
addq $0x120, %rdi            # imm = 0x120
leaq 0x5586(%rip), %rsi      # 0x33a15
callq 0x4350
movq -0x18(%rbp), %rax
movl $0x4, (%rax)
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
addq $0x3, %rdx
movl $0x1, %ecx
callq 0x2e7b0
movl %eax, -0x4(%rbp)
jmp 0x2e681
jmp 0x2e4c2
movl $0x0, -0x30(%rbp)
movl -0x30(%rbp), %eax
movq -0x10(%rbp), %rcx
cmpl 0x14(%rcx), %eax
jge 0x2e57b
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %esi
movq -0x10(%rbp), %rax
movslq -0x30(%rbp), %rcx
movq 0x20(%rax,%rcx,8), %rdx
callq 0x2de40
cmpl $0x0, %eax
je 0x2e56b
movq -0x10(%rbp), %rax
movslq -0x30(%rbp), %rcx
movq 0x20(%rax,%rcx,8), %rax
movq %rax, -0x38(%rbp)
movq -0x38(%rbp), %rax
cmpb $0x0, (%rax)
je 0x2e537
movq -0x10(%rbp), %rdi
movq -0x20(%rbp), %rsi
movq -0x38(%rbp), %rax
movsbl (%rax), %edx
callq 0x2df10
movq %rax, -0x20(%rbp)
movq -0x38(%rbp), %rax
addq $0x1, %rax
movq %rax, -0x38(%rbp)
jmp 0x2e508
movq -0x20(%rbp), %rax
movb $0x0, (%rax)
movl -0x30(%rbp), %ecx
addl $0x3e8, %ecx            # imm = 0x3E8
movq -0x18(%rbp), %rax
movl %ecx, (%rax)
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
movl $0x1, %ecx
callq 0x2e7b0
movl %eax, -0x4(%rbp)
jmp 0x2e681
jmp 0x2e56d
movl -0x30(%rbp), %eax
addl $0x1, %eax
movl %eax, -0x30(%rbp)
jmp 0x2e4c9
jmp 0x2e57d
movq -0x10(%rbp), %rdi
movq -0x20(%rbp), %rsi
movl -0x24(%rbp), %edx
callq 0x2df10
movq %rax, -0x20(%rbp)
movq -0x20(%rbp), %rax
movb $0x0, (%rax)
cmpl $0x22, -0x24(%rbp)
je 0x2e5a4
cmpl $0x27, -0x24(%rbp)
jne 0x2e5d3
movq -0x10(%rbp), %rax
movl 0x10(%rax), %eax
andl $0x1, %eax
cmpl $0x0, %eax
je 0x2e5d1
movq -0x10(%rbp), %rdi
movl -0x24(%rbp), %eax
movq -0x18(%rbp), %rdx
movl -0x2c(%rbp), %ecx
movsbl %al, %esi
callq 0x2e830
movl %eax, -0x4(%rbp)
jmp 0x2e681
jmp 0x2e5d3
movq -0x18(%rbp), %rax
movl $0x9, (%rax)
movl -0x24(%rbp), %ecx
movq -0x18(%rbp), %rax
movl %ecx, 0xc(%rax)
cmpl $0xa, -0x24(%rbp)
jne 0x2e624
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
movl $0x1, %ecx
callq 0x2e7b0
movq -0x10(%rbp), %rax
movl 0x8(%rax), %ecx
addl $0x1, %ecx
movl %ecx, 0x8(%rax)
movq -0x10(%rbp), %rax
movl $0x0, 0xc(%rax)
movl $0x1, -0x4(%rbp)
jmp 0x2e681
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
movl $0x1, %ecx
callq 0x2e7b0
movl %eax, -0x4(%rbp)
jmp 0x2e681
movq -0x20(%rbp), %rax
movb $0x0, (%rax)
movq -0x10(%rbp), %rdi
addq $0x120, %rdi            # imm = 0x120
callq 0x2ea40
movl %eax, %ecx
movq -0x18(%rbp), %rax
movl %ecx, (%rax)
movq -0x10(%rbp), %rdi
movq -0x18(%rbp), %rsi
movq -0x20(%rbp), %rdx
movq -0x18(%rbp), %rax
cmpl $0xa, (%rax)
setne %al
andb $0x1, %al
movzbl %al, %ecx
callq 0x2e7b0
movl %eax, -0x4(%rbp)
movl -0x4(%rbp), %eax
addq $0x50, %rsp
popq %rbp
retq
nopw (%rax,%rax)