// File: /workspace/github_c_makefile_star2_O3/sisshiki1969[P]monocc/src/tokenize.c
// Function: tokenize
void tokenize(char *file, char *p, bool is_main) {
  Token head;
  head.next = NULL;
  Token *cur = &head;
  FileInfo *fi = new_file_info(file, p);
  TokContext *ctx = new_tok_context(fi, cur, p);

  while (*p) {
    skip_whitespace(&p);

    if (*p == '#') {
      char *line_end = strchr(p++, '\n');
      skip_whitespace(&p);
      int len = 0;
      while (is_ident_char(*(p + len))) len++;

      if (is_reserved(p, len, "include")) {
        p += len;
        skip_whitespace(&p);
        if (*p == '"') {
          // get directory name of the current file.
          char *cur_dir;
          char *f = strrchr(file, '/');
          if (!f)
            cur_dir = "./";
          else {
            cur_dir = calloc(1, f - file + 2);
            strncpy(cur_dir, file, f - file + 1);
          }
          // fprintf(stderr, "directory: %s\n", cur_dir);
          // concat the include file name to the directory name.
          char *end = strchr(p + 1, '"');
          int len = end - p - 1;
          char *file_name = calloc(1, strlen(cur_dir) + len + 1);
          strcpy(file_name, cur_dir);
          strncpy(file_name + strlen(cur_dir), p + 1, len);

          // fprintf(stderr, "include <%s>\n", file_name);
          char *text = read_file(file_name);
          tokenize(file_name, text, false);
        }
      } else if (is_reserved(p, len, "define")) {
        // fprintf(stderr, "macro define: ");
        p += len;
        skip_whitespace(&p);
        if (!isalpha(*p) && *p != '_')
          error_at_char(fi, p, "Expected identifier.");
        int len = 0;
        while (is_ident_char(p[len])) len++;

        Token *head = cur;
        Token *tok = new_token(TK_IDENT, cur, p, len);
        p += len;

        ctx->p = p;
        ctx->cur = head;
        // print_token(stderr, tok);
        head->next = NULL;

        if (read_if(ctx, '(')) {
          int i = 0;
          // read params of function-like macro
          // fprintf(stderr, " ( ");
          skip_whitespace(&ctx->p);
          // function macro
          while (*ctx->p != ')') {
            Token *param_token = read_token(ctx);
            param_token->num_val = i++;
            // print_token(stderr, param_token);

            skip_whitespace(&ctx->p);
            if (*ctx->p != ',') break;
            ctx->p++;
          }
          if (!read_if(ctx, ')')) error_at_char(fi, ctx->p, "Expected ')'");
          // fprintf(stderr, " ) ");
        }

        // fprintf(stderr, "=>");
        Token *params = head->next;
        head->next = NULL;
        ctx->cur = head;

        skip_whitespace(&ctx->p);
        int i = 0;
        while (*ctx->p != '\n' && i < 50) {
          read_token(ctx);
          // print_token(stderr, read_token(ctx));
          skip_whitespace(&ctx->p);
          i++;
        }
        new_macro(tok, params, head->next);
        head->next = NULL;

        // fprintf(stderr, "\n");
      }

      p = line_end + 1;
      continue;
    }

    // TokContext *ctx = new_tok_context(fi, cur, p);
    ctx->p = p;
    ctx->cur = cur;
    while (*ctx->p != '\n' && *ctx->p != 0) {
      read_token(ctx);
    }

    while (*ctx->p == '\n') ctx->p++;
    cur = ctx->cur;
    p = ctx->p;
  }

  if (!token) {
    token = head.next;
  } else {
    Token *cursor = token;
    while (cursor->next) cursor = cursor->next;
    cursor->next = head.next;
  }
  if (is_main) new_token(TK_EOF, ctx->cur, ctx->p, 1);
  fi->end = ctx->p;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x88, %rsp
movl %edx, 0x24(%rsp)
movq %rsi, %r13
leaq 0x58(%rsp), %r14
movq $0x0, 0x8(%r14)
movq %rdi, 0x30(%rsp)
callq 0x951c
movq %rax, %rbx
movl $0x1, %edi
movl $0x18, %esi
callq 0x10d0
movq %rax, %r15
movq %rbx, 0x10(%rsp)
movq %rbx, (%rax)
movq %r14, 0x8(%rax)
movq %r13, 0x10(%rax)
movb (%r13), %al
testb %al, %al
je 0x9bcd
movq %r14, 0x18(%rsp)
movq %r15, 0x8(%rsp)
movq %r13, %rbp
leaq 0x1(%rbp), %r15
leaq 0x2(%rbp), %r14
movq %r14, %r12
cmpb $0x9, %al
je 0x9791
movzbl %al, %eax
cmpl $0x20, %eax
jne 0x97a2
movb 0x1(%rbp), %al
incq %rbp
incq %r15
incq %r12
incq %r14
jmp 0x9785
cmpl $0x23, %eax
jne 0x9b7a
movq %rbp, %rdi
movl $0xa, %esi
callq 0x10a0
movq %rax, 0x50(%rsp)
movzbl (%r15), %eax
cmpl $0x20, %eax
je 0x97cb
cmpl $0x9, %eax
jne 0x97d9
incq %r15
incq %rbp
incq %r12
incq %r14
jmp 0x97bd
callq 0x11c0
movq %rax, 0x38(%rsp)
movq (%rax), %r8
movl $0xffffffff, %esi       # imm = 0xFFFFFFFF
movq %rbp, %rdx
movq %r12, %rcx
movq %r14, %rdi
incq %rbp
movsbq 0x1(%rdx), %rbx
incl %esi
incq %r12
incq %r14
cmpq $0x5f, %rbx
je 0x97eb
movzwl (%r8,%rbx,2), %eax
andl $0x8, %eax
jne 0x97eb
movq %rdi, %r12
movq %r8, 0x48(%rsp)
movq %rdx, 0x40(%rsp)
movq %rcx, 0x28(%rsp)
movq %r15, %rdi
leaq 0x4134(%rip), %rdx      # 0xd964
movl %esi, %r14d
callq 0x94e4
testb %al, %al
je 0x985e
movq 0x8(%rsp), %r15
movq 0x28(%rsp), %r14
cmpb $0x9, %bl
je 0x9853
movzbl %bl, %eax
cmpl $0x20, %eax
jne 0x98ab
movb (%r14), %bl
incq %r12
incq %r14
jmp 0x9846
movq %r15, %rdi
movl %r14d, %esi
leaq 0x4104(%rip), %rdx      # 0xd96f
callq 0x94e4
testb %al, %al
movq 0x8(%rsp), %r15
je 0x9b62
movq 0x28(%rsp), %rcx
movq 0x40(%rsp), %rdx
movq 0x48(%rsp), %rsi
cmpb $0x20, %bl
je 0x989d
movzbl %bl, %eax
cmpl $0x9, %eax
jne 0x996d
movb 0x1(%rbp), %bl
incq %rbp
incq %rdx
incq %rcx
jmp 0x988c
cmpl $0x22, %eax
jne 0x9b62
movq 0x30(%rsp), %rbx
movq %rbx, %rdi
movl $0x2f, %esi
callq 0x10b0
leaq 0x409f(%rip), %rbp      # 0xd96c
testq %rax, %rax
je 0x98fa
movq %rax, %r15
subq %rbx, %r15
leaq 0x2(%r15), %rsi
movl $0x1, %edi
callq 0x10d0
movq %rax, %rbp
incq %r15
movq %rax, %rdi
movq %rbx, %rsi
movq %r15, %rdx
callq 0x1040
movq %r14, %rdi
movl $0x22, %esi
callq 0x10a0
movq %rax, %r15
subl %r12d, %r15d
movq %rbp, %rdi
callq 0x1090
movq %rbp, %r12
movq %rax, %rbp
movslq %r15d, %r15
leaq (%rax,%r15), %rsi
incq %rsi
movl $0x1, %edi
callq 0x10d0
movq %rax, %rbx
movq %rax, %rdi
movq %r12, %rsi
callq 0x1060
addq %rbx, %rbp
movq %rbp, %rdi
movq %r14, %rsi
movq %r15, %rdx
callq 0x1040
movq %rbx, %rdi
callq 0x55e0
movq %rbx, %rdi
movq %rax, %rsi
xorl %edx, %edx
callq 0x970d
movq 0x8(%rsp), %r15
jmp 0x9b62
cmpb $0x5f, %bl
je 0x99ab
movsbq %bl, %rax
movzwl (%rsi,%rax,2), %eax
andl $0x400, %eax            # imm = 0x400
jne 0x99ab
movq %rcx, %rbx
movq 0x10(%rsp), %rdi
movq %rbp, %rsi
movq %rdx, %r14
leaq 0x39eb(%rip), %rdx      # 0xd381
xorl %eax, %eax
callq 0x5204
movq %r14, %rdx
movq %rbx, %rcx
movq 0x38(%rsp), %rax
movq (%rax), %rsi
movl $0xffffffff, %ebx       # imm = 0xFFFFFFFF
movq %rcx, %r15
movsbq 0x1(%rdx), %rax
incq %rdx
incl %ebx
incq %rcx
cmpq $0x5f, %rax
je 0x99b0
movzwl (%rsi,%rax,2), %eax
andl $0x8, %eax
jne 0x99b0
movl $0x1, %edi
movl $0x30, %esi
movq %rdx, %r14
callq 0x10d0
movq %rax, %r12
movl $0x0, (%rax)
movq %rbp, 0x20(%rax)
movl %ebx, 0x28(%rax)
movq 0x8(%rsp), %rax
movq %r14, 0x10(%rax)
movq 0x18(%rsp), %rbp
movq %rbp, 0x8(%rax)
movq $0x0, 0x8(%rbp)
cmpb $0x28, (%r14)
jne 0x9abe
movq 0x8(%rsp), %rax
movq %r15, 0x10(%rax)
movzbl (%r15), %eax
cmpl $0x9, %eax
je 0x9a2c
cmpl $0x20, %eax
jne 0x9a31
incq %r15
jmp 0x9a15
cmpl $0x29, %eax
je 0x9a92
xorl %ebx, %ebx
movq 0x8(%rsp), %r14
movq %r14, %rdi
callq 0x9c3e
movq %rbx, 0x10(%rax)
movq 0x10(%r14), %r15
incq %r15
movzbl -0x1(%r15), %eax
cmpl $0x1f, %eax
jle 0x9a61
cmpl $0x20, %eax
je 0x9a66
jmp 0x9a71
cmpl $0x9, %eax
jne 0x9aa0
movq 0x8(%rsp), %rax
movq %r15, 0x10(%rax)
jmp 0x9a4d
cmpl $0x29, %eax
je 0x9a8f
cmpl $0x2c, %eax
jne 0x9aa0
incq %rbx
movq 0x8(%rsp), %rax
movq %r15, 0x10(%rax)
cmpb $0x29, (%r15)
jne 0x9a38
jmp 0x9a92
decq %r15
incq %r15
movq 0x8(%rsp), %rax
movq %r15, 0x10(%rax)
jmp 0x9abe
decq %r15
movq 0x10(%rsp), %rdi
movq %r15, %rsi
leaq 0x3ec4(%rip), %rdx      # 0xd976
xorl %eax, %eax
callq 0x5204
movq 0x8(%rsp), %rax
movq 0x8(%rbp), %rbx
movq $0x0, 0x8(%rbp)
movq %rbp, 0x8(%rax)
movq 0x10(%rax), %r13
movq %rax, %r15
movzbl (%r13), %eax
cmpl $0x9, %eax
je 0x9ae4
cmpl $0x20, %eax
jne 0x9aed
incq %r13
movq %r13, 0x10(%r15)
jmp 0x9ad5
cmpl $0xa, %eax
je 0x9b2a
xorl %r14d, %r14d
movq %r15, %rdi
callq 0x9c3e
movq 0x10(%r15), %r13
movzbl (%r13), %eax
cmpl $0x20, %eax
je 0x9b10
cmpl $0x9, %eax
jne 0x9b19
incq %r13
movq %r13, 0x10(%r15)
jmp 0x9b01
cmpb $0xa, %al
je 0x9b2a
leal 0x1(%r14), %eax
cmpl $0x31, %r14d
movl %eax, %r14d
jb 0x9af5
movq 0x8(%rbp), %r14
movl $0x1, %edi
movl $0x20, %esi
callq 0x10d0
movq %r12, 0x8(%rax)
movq %rbx, 0x10(%rax)
movq %r14, 0x18(%rax)
movq 0x9730(%rip), %rcx      # 0x13280
movq %rcx, (%rax)
movq %rax, 0x9726(%rip)      # 0x13280
movq $0x0, 0x8(%rbp)
movq 0x50(%rsp), %rax
movq %rax, %rbp
incq %rbp
movb 0x1(%rax), %al
testb %al, %al
jne 0x977a
jmp 0x9bcd
movq 0x8(%rsp), %r15
movq %rbp, 0x10(%r15)
movq 0x18(%rsp), %rax
movq %rax, 0x8(%r15)
movzbl (%rbp), %eax
cmpl $0xa, %eax
je 0x9ba7
testl %eax, %eax
je 0x9ba7
movq %r15, %rdi
callq 0x9c3e
movq 0x10(%r15), %rbp
jmp 0x9b8c
decq %rbp
movq %rbp, %r13
movb 0x1(%r13), %al
incq %r13
cmpb $0xa, %al
je 0x9bad
movq %r13, 0x10(%r15)
movq 0x8(%r15), %rcx
movq %rcx, 0x18(%rsp)
testb %al, %al
jne 0x9777
leaq 0x9654(%rip), %rax      # 0x13228
movq (%rax), %rcx
testq %rcx, %rcx
je 0x9bec
movq %rcx, %rax
movq 0x8(%rcx), %rcx
testq %rcx, %rcx
jne 0x9bdc
addq $0x8, %rax
movq 0x60(%rsp), %rcx
movq %rcx, (%rax)
cmpb $0x0, 0x24(%rsp)
je 0x9c23
movq 0x8(%r15), %rbx
movl $0x1, %edi
movl $0x30, %esi
callq 0x10d0
movl $0x41, (%rax)
movq %r13, 0x20(%rax)
movl $0x1, 0x28(%rax)
movq %rax, 0x8(%rbx)
movq 0x10(%rsp), %rax
movq %r13, 0x18(%rax)
addq $0x88, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq