How to make a language (using Python)
Tags: programming, beginners, tutorial, python
Published: 2022-02-17 23:07:29

After completing this tutorial, you should be able to make a file that looks like this:

```
var age = 18 + 5;
output "You are " + age;

endproc 0;
```

This should do the following:
* Store the "age" variable in the memory, with the value of 23
* Output "You are 23"
* End the process on exit code 0

So, to start off, we should first make a main file. This shouldn't take too long:

```python
# import the parser and lexer
import lexer as l
import parser as p

with open('main.lopa', 'r') as f:
    #################
    #     LEXER     #
    #################
    contents = [i for j in f.read().split() for i in (j, ' ')][:-1] # include spaces in the input file
    lexer = l.Lexer(contents)
    tokens = lexer.tokenize() # tokenize the contents
    
    ################
    #    PARSER    #
    ################
    parser = p.Parser(tokens)
    parser.parse() # parse the tokenized contents from the lexer
    parser.generateFile('main.py') # generate a python file after the parsing has been completed, this is what the user will run
```

Once you have completed that, it is time to move on to the lexer!

```python
import re # import regex

class Lexer:
    def __init__(self, source):
        self.source = source
    
    # tokenize function
    def tokenize(self):
        # initalize variables
        tokens = [] # the tokens array, this is returned after tokenization
        index = 0 # the index the lexer is on of the inputs

        while index < len(self.source):
            word = self.source[index]

            # this already contains a few things from my own language, but i will describe what it does anyways and how to utilize it
            # to add on to this lexer, all you have to do is simply check if a word equals something, if it does then append it to the tokens list with an array with two values (name, literal)

            if word == 'set':
                tokens.append(['VARIABLE_DECLARATOR', word])

            elif re.match('[a-zA-Z]', word):
                if word[len(word) - 1] == ';':
                    tokens.append(['IDENTIFIER', word[0:len(word) - 1]])
                else:
                    tokens.append(['IDENTIFIER', word])

            elif word[0] == '"':
                buffer = []

                if word[len(word) - 1] != ';':
                    while word[len(word) - 1] != '"':
                        buffer.append(word)
                        
                        index += 1
                        word = self.source[index]

                    buffer.append(word)
                else:
                    buffer.append(word[0:len(word) - 1])

                tokens.append(['STRING', ''.join(buffer)])

            elif re.match('-?[0-9]', word):
                if word[len(word) - 1] == ';':
                    tokens.append(['NUMBER', word[0:len(word) - 1]])
                else:
                    tokens.append(['NUMBER', word])
            elif word in '=/*-+{}()':
                tokens.append(['OPERATOR', word])

            if word[len(word) -1] == ";":
                tokens.append(['END', ';'])

            # increment the index - this should not be in the if statements
            index += 1
        
        # finally, return the tokens, on to parsing!
        return tokens
```

After all of that, it's time to parse!

```python
import lexer as l

class Parser:
    def __init__(self, tokens):
        self.tokens = tokens
        self.index = 0
        self.transpiled = '' # you can include a watermark, if you'd like. anything that is transpiled will be appended to this string

    # this function will loop over the tokenized contents and determine what to parse
    def parse(self):
        while self.index < len(self.tokens):
            # stores token types
            t_type  = self.tokens[self.index][0]

            # stores value of token
            t_value = self.tokens[self.index][1]

            if t_type == "VARIABLE_DECLARATOR" and t_value == 'set':
                self.parse_variable_declaration(self.tokens[self.index:len(self.tokens)])

            elif t_type == "IDENTIFIER" and t_value == 'output':
                self.parse_output_statement(self.tokens[self.index:len(self.tokens)])
            elif t_type == "IDENTIFIER" and t_value == 'endproc':
                self.parse_endproc(self.tokens[self.index:len(self.tokens)])

            self.index += 1
    
    # this function will generate the file
    def generateFile(self, output):
        with open(output, 'w') as f:
            f.write(self.transpiled)

    # the parser functions

    # variables
    def parse_variable_declaration(self, stream):
        check = 0

        # these 3 variables determine the name, operator, and value of the variable (eg: +-/*...)
        name     = ''
        operator = ''
        value    = ''

        for token in range(0, len(stream)):
            t_type = stream[check][0]
            t_value = stream[check][1]

            if t_type == 'END':
                break

            elif token == 1 and t_type == 'IDENTIFIER':
                name = t_value
            elif token == 1 and t_type != 'IDENTIFIER':
                print(f'ERR -> Failed to parse, invalid variable name \'{t_value}\'')
                quit(-1)

            elif token == 2 and t_type == 'OPERATOR':
                operator = t_value
            elif token == 2 and t_type != 'OPERATOR':
                print('ERR -> Failed to parse, assignment operator is missing or invalid on the declaration of a variable')
                quit(-1)

            elif token > 2 and t_type in ['STRING', 'NUMBER', 'IDENTIFIER', 'OPERATOR']:
                value += t_value
            elif token > 2 and t_type not in ['STRING', 'NUMBER', 'IDENTIFIER', 'OPERATOR']:
                print(f'ERR -> Failed to parse, invalid assignment value, {t_value}')
                quit(-1)

            check += 1

        # finally, push the transpiled code
        self.transpiled += f'{name} {operator} {value}\n'
        self.index += check

    # everything else is basically the same, i recommend to seperate these all into different files to avoid bloating up your file and making it unreadable. happy coding!

    # output statement
    def parse_output_statement(self, stream):
        check = 0
        value = ''

        for token in range(0, len(stream)):
            t_type = stream[check][0]
            t_value = stream[check][1]

            if t_type == 'END':
                break

            elif token > 0 and t_type in ['STRING', 'NUMBER', 'IDENTIFIER', 'OPERATOR']:
                value += t_value
            elif token > 0 and t_type not in ['NUMBER', 'IDENTIFIER', 'OPERATOR']:
                print(f'ERR -> Failed to parse, invalid assignment value, {t_value}')
                quit(-1)

            check += 1

        self.transpiled += f'print({value})\n'
        self.index += check

    # endproc
    def parse_endproc(self, stream):
        check = 0
        code = 0

        for token in range(0, len(stream)):
            t_type = stream[check][0]
            t_value = stream[check][1]

            if t_type == 'END':
                break

            elif token == 1 and t_type == 'NUMBER':
                if '.' in t_value:
                    print('ERR -> Failed to parse, cannot quit program on a decimal number')
                    quit(-1)

                code = t_value
            elif token == 1 and t_type not in ['NUMBER', None]:
                print(f'ERR -> Failed to parse, cannot parse value {t_value} on \'endproc\'')
                quit(-1)

            check += 1

        self.transpiled += f'quit({code})\n'
        self.index += check
```