global error_det
error_det = ''

import string
from strings_with_arrows import *

from errors import *
from nodes import *

DIGITS = '0123456789'
LETTERS = string.ascii_letters
LETTERS_DIGITS = LETTERS + DIGITS

class Position:
    def __init__(self,idx,ln,col,fn,ftxt):
        self.fn = fn
        self.ftxt = ftxt
        self.idx = idx
        self.ln = ln
        self.col = col

    def advance(self,current_char = None):
        self.idx += 1
        self.col += 1

        if current_char == '\n':
            self.ln += 1
            self.col = 0

        return self
    def regress(self,current_char = None):
        self.idx -= 1
        self.col -= 1

    def copy(self):
        return Position(self.idx,self.ln,self.col,self.fn,self.ftxt)

TT_INT = 'Int'
TT_REAL = 'Real'
TT_STRING = 'String'
TT_BOOLEAN = 'Boolean'
TT_CHAR = 'Char'
TT_IDENTIFIER = 'Identifier'
TT_KEYWORD = 'Keyword'
TT_EQ = 'Eq'
TT_PLUS = 'Plus'
TT_MINUS = 'Minus'
TT_MUL = 'Mul'
TT_DIV = 'Div'
TT_POWER = 'Power'
TT_LPAREN = 'LParen'
TT_RPAREN = "RParen"
TT_LSQUAREBRACKET = 'LSquareBracket'
TT_RSQUAREBRACKET = 'RSquareBracket'
TT_LT = 'NE'
TT_EE = 'EE'
TT_LT = 'LT'
TT_GT = 'GT'
TT_LTE = 'LTE'
TT_GTE = 'GTE'
TT_NE = 'NE'
TT_Comma = 'COMMA'
TT_Arrow = "ARROW"
TT_Newline = 'Newline'
TT_EOF = "Eof"
TT_ARRAY = 'ARRAY'
TT_PROCEDURE = 'PROCEDURE'
TT_END_PROCEDURE = 'ENDPROCEDURE'

KEYWORDS = [
'AND',"OR","NOT","IF","ELSE","THEN","FOR","WHILE","DO","STEP","TO","REPEAT","UNTIL","FUNCTION",'RETURNS','ENDIF','ENDFUNCTION','RETURNS','ENDWHILE',"NEXT",'RETURN', 'PROCEDURE', 'ARRAY'
]

ALLOWED_ARRAY_TYPES = ['INTEGER', 'REAL', 'STRING', 'BOOLEAN', 'CHAR']

class Token:
    def __init__(self,type_,value = None, pos_start = None, pos_end = None):
        self.type = type_
        self.value = value
        if pos_start:
            self.pos_start = pos_start
            self.pos_end = self.pos_start.copy()
            self.pos_end.advance()
        if pos_end:
            self.pos_end = pos_end
    def __repr__(self):
        if self.value: return f'{self.type}:{self.value}'
        return f'{self.type}'

    def matches(self,type_,value):
        return self.type == type_ and self.value == value
    
class Lexer:
    def __init__(self,fn, text):
        self.fn = fn
        self.text = text
        self.pos = Position(-1,0,-1,fn,text)
        self.current_char = None
        self.advance()
    def advance(self):
        self.pos.advance(self.current_char)
        self.current_char = self.text[self.pos.idx] if self.pos.idx < len(self.text) else None
    def regress(self):
        self.pos.regress(self.current_char)
        self.current_char = self.text[self.pos.idx] if self.pos.idx < len(self.text) else None

    def make_array(self):
        pos_start = self.pos.copy()
        self.advance()  # Skip the '['

        min_value = self.make_expr()

        if not min_value:
            return [], InvalidSyntaxError(pos_start, self.pos, "Expected a number or expression after '['")

        if self.current_char != ':':
            return [], InvalidSyntaxError(pos_start, self.pos, "Expected ':' in array declaration")

        self.advance()  # Skip the ':'

        max_value = self.make_expr()

        if not max_value:
            return [], InvalidSyntaxError(pos_start, self.pos, "Expected a number or expression after ':' in array declaration")

        if self.current_char != ']':
            return [], InvalidSyntaxError(pos_start, self.pos, "Expected ']' in array declaration")

        self.advance()  # Skip the ']'

        if self.current_char != 'OF':
            return [], InvalidSyntaxError(pos_start, self.pos, "Expected 'OF' in array declaration")

        self.advance()  # Skip 'OF'

        array_type = self.make_identifier()

        if array_type not in ALLOWED_ARRAY_TYPES:
            return [], InvalidSyntaxError(pos_start, self.pos, f"Expected one of {ALLOWED_ARRAY_TYPES} in array declaration")

        return [Token(TT_ARRAY, (min_value, max_value, array_type), pos_start, self.pos)], None


    def make_tokens(self):
        tokens = []

        while self.current_char is not None:

            if self.current_char in ' \t':
                self.advance()

            elif self.current_char in ['\n', ';']:
                tokens.append(Token(TT_Newline, pos_start=self.pos))
                self.advance()

            elif self.current_char in LETTERS:
                tokens.append(self.make_identifier())

            elif self.current_char in DIGITS:
                tokens.append(self.make_number())

            elif self.current_char in ['"', "'"]:
                tokens.append(self.make_string(self.current_char))

            elif self.current_char == '[':
                array_tokens, error = self.make_array()
                if error:
                    return [], error
                tokens += array_tokens

            elif self.current_char == ']':
                tokens.append(Token(TT_RSQUAREBRACKET, pos_start=self.pos))
                self.advance()

            elif self.current_char == '+':
                tokens.append(Token(TT_PLUS, pos_start=self.pos))
                self.advance()

            elif self.current_char == '-':
                tokens.append(Token(TT_MINUS, pos_start=self.pos))
                self.advance()

            elif self.current_char == ':':
                tokens.append(Token(TT_Arrow, pos_start=self.pos))
                self.advance()

            elif self.current_char == '*':
                tokens.append(Token(TT_MUL, pos_start=self.pos))
                self.advance()

            elif self.current_char == '/':
                tokens.append(Token(TT_DIV, pos_start=self.pos))
                self.advance()

            elif self.current_char == '(':
                tokens.append(Token(TT_LPAREN, pos_start=self.pos))
                self.advance()

            elif self.current_char == ')':
                tokens.append(Token(TT_RPAREN, pos_start=self.pos))
                self.advance()

            elif self.current_char == '^':
                tokens.append(Token(TT_POWER, pos_start=self.pos))
                self.advance()

            elif self.current_char == '<':
                tok = self.make_not_lessThan_assign()
                tokens.append(tok)

            elif self.current_char == '>':
                tokens.append(self.make_greater_than())

            elif self.current_char == ',':
                tokens.append(Token(TT_Comma, pos_start=self.pos))
                self.advance()

            elif self.current_char == 'P':
                tok = self.make_keyword()
                if tok.value == 'PROCEDURE':
                    tokens.append(Token(TT_PROCEDURE, pos_start=tok.pos_start, pos_end=tok.pos_end))
                elif tok.value == 'ENDPROCEDURE':
                    tokens.append(Token(TT_END_PROCEDURE, pos_start=tok.pos_start, pos_end=tok.pos_end))
                else:
                    return [], IllegalCharError(tok.pos_start, tok.pos_end, f"Unexpected character '{self.current_char}'")

            else:
                pos_start = self.pos.copy()
                char = self.current_char
                self.advance()
                return [], IllegalCharError(pos_start, self.pos, "'" + char + "'")

        tokens.append(Token(TT_EOF, pos_start=self.pos))

        return tokens, None

    def make_string(self,end):
        string = ''
        pos_start = self.pos.copy()
        self.advance()
        while self.current_char != None and self.current_char != end:
            string += self.current_char
            self.advance()

        self.advance()
        return Token(TT_STRING,string,pos_start,self.pos)


    def make_number(self):
        num_str = ''
        dotCount = 0
        pos_start = self.pos.copy()
        while self.current_char != None and self.current_char in DIGITS + '.':
            if self.current_char == '.':
                if dotCount == 1: break
                num_str += '.'
                dotCount += 1
            else:
                num_str += self.current_char
            self.advance()
        if dotCount == 0:
            return Token(TT_INT,int(num_str),pos_start,self.pos)
        else:
            return Token(TT_REAL,float(num_str),pos_start,self.pos)

    def make_identifier(self):
        id_str = ''
        pos_start = self.pos.copy()

        while self.current_char != None and self.current_char in LETTERS_DIGITS + '_':
            id_str += self.current_char
            self.advance()
        tok_type = TT_KEYWORD if id_str in KEYWORDS else TT_IDENTIFIER

        return Token(tok_type,id_str,pos_start,self.pos)

    def make_greater_than(self):
        pos_start = self.pos.copy()
        self.advance()
        if self.current_char == '=':
            self.advance()
            return Token(TT_GTE,pos_start=pos_start,pos_end=self.pos)
        return Token(TT_GT,pos_start=pos_start,pos_end=self.pos)

    def make_not_lessThan_assign(self):
        pos_start = self.pos.copy()
        self.advance()
        if self.current_char == '-':
            self.advance()
            return Token(TT_EQ,pos_start=pos_start,pos_end=self.pos)
        elif self.current_char == '>':
            self.advance()
            return Token(TT_NE,pos_start=pos_start,pos_end=self.pos)
        elif self.current_char == '=':
            self.advance()
            return Token(TT_LTE, pos_start=pos_start, pos_end=self.pos)
        else:
            return Token(TT_LT, pos_start=pos_start, pos_end=self.pos)


so far these are my tokens and lexer for a pseudocode engine that has arrays and procedures and the other stuff.