C program to detect tokens in a C program - GeeksforGeeks

15 Jul, 2021
As it is known that Lexical Analysis is the first phase of compiler also known as scanner. It converts the input program into a sequence of Tokens. A C program consists of various tokens and a token is either a keyword, an identifier, a constant, a string literal, or a symbol.For Example: 
1) Keywords: 
Examples- for, while, if etc.

2) Identifier
Examples- Variable name, function name etc.

3) Operators:
Examples- '+', '++', '-' etc.

4) Separators:
Examples- ', ' ';' etc
Below is a C program to print all the keywords, literals, valid identifiers, invalid identifiers, integer number, real number in a given C program:
C
#include <stdbool.h>#include <stdio.h>#include <string.h>#include <stdlib.h> // Returns 'true' if the character is a DELIMITER.bool isDelimiter(char ch){    if (ch == ' ' || ch == '+' || ch == '-' || ch == '*' ||        ch == '/' || ch == ',' || ch == ';' || ch == '>' ||        ch == '<' || ch == '=' || ch == '(' || ch == ')' ||        ch == '[' || ch == ']' || ch == '{' || ch == '}')        return (true);    return (false);} // Returns 'true' if the character is an OPERATOR.bool isOperator(char ch){    if (ch == '+' || ch == '-' || ch == '*' ||        ch == '/' || ch == '>' || ch == '<' ||        ch == '=')        return (true);    return (false);} // Returns 'true' if the string is a VALID IDENTIFIER.bool validIdentifier(char* str){    if (str[0] == '0' || str[0] == '1' || str[0] == '2' ||        str[0] == '3' || str[0] == '4' || str[0] == '5' ||        str[0] == '6' || str[0] == '7' || str[0] == '8' ||        str[0] == '9' || isDelimiter(str[0]) == true)        return (false);    return (true);} // Returns 'true' if the string is a KEYWORD.bool isKeyword(char* str){    if (!strcmp(str, "if") || !strcmp(str, "else") ||        !strcmp(str, "while") || !strcmp(str, "do") ||        !strcmp(str, "break") ||         !strcmp(str, "continue") || !strcmp(str, "int")        || !strcmp(str, "double") || !strcmp(str, "float")        || !strcmp(str, "return") || !strcmp(str, "char")        || !strcmp(str, "case") || !strcmp(str, "char")        || !strcmp(str, "sizeof") || !strcmp(str, "long")        || !strcmp(str, "short") || !strcmp(str, "typedef")        || !strcmp(str, "switch") || !strcmp(str, "unsigned")        || !strcmp(str, "void") || !strcmp(str, "static")        || !strcmp(str, "struct") || !strcmp(str, "goto"))        return (true);    return (false);} // Returns 'true' if the string is an INTEGER.bool isInteger(char* str){    int i, len = strlen(str);     if (len == 0)        return (false);    for (i = 0; i < len; i++) {        if (str[i] != '0' && str[i] != '1' && str[i] != '2'            && str[i] != '3' && str[i] != '4' && str[i] != '5'            && str[i] != '6' && str[i] != '7' && str[i] != '8'            && str[i] != '9' || (str[i] == '-' && i > 0))            return (false);    }    return (true);} // Returns 'true' if the string is a REAL NUMBER.bool isRealNumber(char* str){    int i, len = strlen(str);    bool hasDecimal = false;     if (len == 0)        return (false);    for (i = 0; i < len; i++) {        if (str[i] != '0' && str[i] != '1' && str[i] != '2'            && str[i] != '3' && str[i] != '4' && str[i] != '5'            && str[i] != '6' && str[i] != '7' && str[i] != '8'            && str[i] != '9' && str[i] != '.' ||            (str[i] == '-' && i > 0))            return (false);        if (str[i] == '.')            hasDecimal = true;    }    return (hasDecimal);} // Extracts the SUBSTRING.char* subString(char* str, int left, int right){    int i;    char* subStr = (char*)malloc(                  sizeof(char) * (right - left + 2));     for (i = left; i <= right; i++)        subStr[i - left] = str[i];    subStr[right - left + 1] = '\0';    return (subStr);} // Parsing the input STRING.void parse(char* str){    int left = 0, right = 0;    int len = strlen(str);     while (right <= len && left <= right) {        if (isDelimiter(str[right]) == false)            right++;         if (isDelimiter(str[right]) == true && left == right) {            if (isOperator(str[right]) == true)                printf("'%c' IS AN OPERATOR\n", str[right]);             right++;            left = right;        } else if (isDelimiter(str[right]) == true && left != right                   || (right == len && left != right)) {            char* subStr = subString(str, left, right - 1);             if (isKeyword(subStr) == true)                printf("'%s' IS A KEYWORD\n", subStr);             else if (isInteger(subStr) == true)                printf("'%s' IS AN INTEGER\n", subStr);             else if (isRealNumber(subStr) == true)                printf("'%s' IS A REAL NUMBER\n", subStr);             else if (validIdentifier(subStr) == true                     && isDelimiter(str[right - 1]) == false)                printf("'%s' IS A VALID IDENTIFIER\n", subStr);             else if (validIdentifier(subStr) == false                     && isDelimiter(str[right - 1]) == false)                printf("'%s' IS NOT A VALID IDENTIFIER\n", subStr);            left = right;        }    }    return;} // DRIVER FUNCTIONint main(){     // maximum length of string is 100 here    char str[100] = "int a = b + 1c; ";     parse(str); // calling the parse function     return (0);}
Output: 
'int' IS A KEYWORD
'a' IS A VALID IDENTIFIER
'=' IS AN OPERATOR
'b' IS A VALID IDENTIFIER
'+' IS AN OPERATOR
'1c' IS NOT A VALID IDENTIFIER
This article is contributed by MAZHAR IMAM KHAN. If you like GeeksforGeeks and would like to contribute, you can also write an article using write.geeksforgeeks.org or mail your article to review-team@geeksforgeeks.org. See your article appearing on the GeeksforGeeks main page and help other Geeks.Please write comments if you find anything incorrect, or you want to share more information about the topic discussed above. 
adnanirshad158
C Language
Compiler Design
Writing code in comment?
Please use ide.geeksforgeeks.org,
generate link and share the link here.
Arrays in C/C++
Dynamic Memory Allocation in C using malloc(), calloc(), free() and realloc()
std::sort() in C++ STL
Bitwise Operators in C/C++
Multidimensional Arrays in C / C++
Construction of LL(1) Parsing Table
Directed Acyclic graph in Compiler Design (with examples)
Types of Parsers in Compiler Design
Difference between Compiler and Interpreter
Peephole Optimization in Compiler Design