Repository: tusharsadhwani/json_parser

Issue:
Potential improvements
- Instead of passing a deque of strings to the parser and making the parser guess the type of each token, pass a type parameter with it in the token object
- Add more complex passing (and especially failing) tests, to ensure proper error messages
- Add line number and column number to error message. (Possibly by adding those values in each token object).

Discussion:

Patch:
diff --git a/json_parser/lexer.py b/json_parser/lexer.py
index 5da0253..ef22b55 100644
--- a/json_parser/lexer.py
+++ b/json_parser/lexer.py
@@ -1,18 +1,46 @@
 """Lexer functions"""
-import string
 from collections import deque
-from typing import Deque
+from typing import Deque, Literal, NamedTuple, Tuple
+from string import whitespace as WHITESPACE
+
+TokenType = Literal[
+    'string',
+    'number',
+    'boolean',
+    'null',
+    'left_bracket',
+    'left_brace',
+    'right_bracket',
+    'right_brace',
+    'comma',
+    'colon',
+]
+
+
+class Token(NamedTuple):
+    """Represents a Token extracted by the parser"""
+    value: str
+    type: TokenType
+    line: int
+    column: int
 
 
 class TokenizeError(Exception):
     """Error thrown when an invalid JSON string is tokenized"""
 
 
-def extract_string(json_string: str, index: int, tokens: Deque[str]) -> int:
+def extract_string(
+        json_string: str,
+        index: int,
+        tokens: Deque[Token],
+        line: int,
+        column: int) -> Tuple[int, int, int]:
     """Extracts a single string token from JSON string"""
     start = index
     end = len(json_string)
+    start_line, start_column = line, column
     index += 1
+    column += 1
 
     while index < end:
         char = json_string[index]
@@ -22,20 +50,34 @@ def extract_string(json_string: str, index: int, tokens: Deque[str]) -> int:
                 raise TokenizeError("Incomplete escape at end of string")
 
             index += 2
+            column += 2
             continue
 
         if char == '"':
             index += 1
-            string_token = json_string[start:index]
-            tokens.append(string_token)
-            return index
+            column += 1
+            string = json_string[start:index]
+            tokens.append(Token(string, 'string', start_line, start_column))
+
+            return index, line, column
 
         index += 1
+        if char == '\n':
+            column = 1
+            line += 1
+        else:
+            column += 1
 
-    raise TokenizeError("Expected end of string")
+    err = f"Expected end of string (line {line} column {column})"
+    raise TokenizeError(err)
 
 
-def extract_number(json_string: str, index: int, tokens: Deque[str]) -> int:
+def extract_number(
+        json_string: str,
+        index: int,
+        tokens: Deque[Token],
+        line: int,
+        column: int) -> Tuple[int, int, int]:
     """Extracts a single number token (eg. 42, -12.3) from JSON string"""
     start = index
     end = len(json_string)
@@ -62,12 +104,19 @@ def extract_number(json_string: str, index: int, tokens: Deque[str]) -> int:
 
         index += 1
 
-    string_token = json_string[start:index]
-    tokens.append(string_token)
-    return index
+    number = json_string[start:index]
+    tokens.append(Token(number, 'number', line, column))
+    length = index - start
+    column += length
+    return index, line, column
 
 
-def extract_special(json_string: str, index: int, tokens: Deque[str]) -> int:
+def extract_special(
+        json_string: str,
+        index: int,
+        tokens: Deque[Token],
+        line: int,
+        column: int) -> Tuple[int, int, int]:
     """Extracts true, false and null from JSON string"""
     end = len(json_string)
 
@@ -81,37 +130,67 @@ def extract_special(json_string: str, index: int, tokens: Deque[str]) -> int:
         index += 1
 
     if word in ('true', 'false', 'null'):
-        tokens.append(word)
-        return index
+        token = Token(
+            word,
+            line=line,
+            column=column,
+            type='null' if word == 'null' else 'boolean'
+        )
+        tokens.append(token)
+        column += len(word)
+        return index, line, column
 
-    raise TokenizeError(f"Unknown token found: {word}")
+    err = f"Unknown token found: {word} (line {line} column {column})"
+    raise TokenizeError(err)
 
 
-def tokenize(json_string: str) -> Deque[str]:
+def tokenize(json_string: str) -> Deque[Token]:
     """Converts a JSON string into a queue of tokens"""
-    tokens: Deque[str] = deque()
+    tokens: Deque[Token] = deque()
 
+    line = column = 1
     index = 0
     end = len(json_string)
     while index < end:
         char = json_string[index]
 
-        if char in string.whitespace:
+        if char in WHITESPACE:
             index += 1
 
-        elif char in '[]{}:,':
-            tokens.append(char)
+            if char == '\n':
+                column = 1
+                line += 1
+            else:
+                column += 1
+
+        elif char in '[]{},:':
+            token = Token(
+                char,
+                line=line,
+                column=column,
+                type=('left_bracket' if char == '[' else
+                      'right_bracket' if char == ']' else
+                      'left_brace' if char == '{' else
+                      'right_brace' if char == '}' else
+                      'comma' if char == ',' else 'colon')
+            )
+            tokens.append(token)
             index += 1
+            column += 1
 
         elif char == '"':
-            index = extract_string(json_string, index, tokens)
+            index, line, column = extract_string(
+                json_string, index, tokens, line, column)
 
         elif char == '-' or char.isdigit():
-            index = extract_number(json_string, index, tokens)
+            index, line, column = extract_number(
+                json_string, index, tokens, line, column)
 
         else:
-            index = extract_special(json_string, index, tokens)
+            index, line, column = extract_special(
+                json_string, index, tokens, line, column)
 
     if len(tokens) == 0:
         raise TokenizeError("Cannot parse empty string")
+
     return tokens
diff --git a/json_parser/parser.py b/json_parser/parser.py
index 960f6be..b195c5f 100644
--- a/json_parser/parser.py
+++ b/json_parser/parser.py
@@ -2,7 +2,7 @@
 from ast import literal_eval
 from typing import Deque, Dict, List, Union
 
-from json_parser.lexer import tokenize
+from json_parser.lexer import Token, tokenize
 
 JSONArray = List[object]
 JSONObject = Dict[str, object]
@@ -13,115 +13,142 @@ class ParseError(Exception):
     """Error thrown when invalid JSON tokens are parsed"""
 
 
-def parse_object(tokens: Deque[str]) -> JSONObject:
+def parse_object(tokens: Deque[Token]) -> JSONObject:
     """Parses an object out of JSON tokens"""
     obj: JSONObject = {}
 
     # special case:
-    if tokens[0] == '}':
+    if tokens[0].type == 'right_brace':
         tokens.popleft()
         return obj
 
     while tokens:
         token = tokens.popleft()
 
-        if not token.startswith('"'):
-            raise ParseError(f"Expected string key for object, found {token}")
+        if not token.type == 'string':
+            raise ParseError(
+                f"Expected string key for object, found {token.value} "
+                f"(line {token.line} column {token.column})")
 
         key = parse_string(token)
 
         if len(tokens) == 0:
-            raise ParseError("Unexpected end of file while parsing")
+            raise ParseError(
+                "Unexpected end of file while parsing (line {line} column {column})")
 
         token = tokens.popleft()
-        if token != ':':
-            raise ParseError(f"Expected colon, found {token}")
+        if token.type != 'colon':
+            raise ParseError(
+                f"Expected colon, found {token.value} "
+                f"(line {token.line} column {token.column})")
 
         # Missing value for key
         if len(tokens) == 0:
-            raise ParseError("Unexpected end of file while parsing")
+            raise ParseError(
+                "Unexpected end of file while parsing "
+                f"(line {token.line} column {token.column+1})")
 
-        if tokens[0] == '}':
-            raise ParseError("Expected value after colon, found }")
+        if tokens[0].type == 'right_brace':
+            token = tokens[0]
+            raise ParseError(
+                "Expected value after colon, found } "
+                f"(line {token.line} column {token.column})")
 
         value = _parse(tokens)
         obj[key] = value
 
         if len(tokens) == 0:
-            raise ParseError("Unexpected end of file while parsing")
+            column_end = token.column + len(token.value)
+            raise ParseError(
+                "Unexpected end of file while parsing "
+                f"(line {token.line} column {column_end})")
 
         token = tokens.popleft()
-        if token not in ',}':
-            raise ParseError(f"Expected ',' or '}}', found {token}")
+        if token.type not in ('comma', 'right_brace'):
+            raise ParseError(
+                f"Expected ',' or '}}', found {token.value}"
+                f" (line {token.line} column {token.column})")
 
-        if token == '}':
+        if token.type == 'right_brace':
             break
 
-        # Trailing comma check
+        # Trailing comma checks
         if len(tokens) == 0:
-            raise ParseError("Unexpected end of file while parsing")
+            column_end = token.column + len(token.value)
+            raise ParseError(
+                "Unexpected end of file while parsing "
+                f"(line {token.line} column {column_end})")
 
-        if tokens[0] == '}':
-            raise ParseError("Expected value after comma, found }")
+        if tokens[0].type == 'right_brace':
+            token = tokens[0]
+            raise ParseError(
+                "Expected value after comma, found } "
+                f"(line {token.line} column {token.column})")
 
     return obj
 
 
-def parse_array(tokens: Deque[str]) -> JSONArray:
+def parse_array(tokens: Deque[Token]) -> JSONArray:
     """Parses an array out of JSON tokens"""
     array: JSONArray = []
 
     # special case:
-    if tokens[0] == ']':
+    if tokens[0].type == 'right_bracket':
         tokens.popleft()
         return array
 
     while tokens:
-        # least number of tokens left should be a token and a ]
-        if len(tokens) < 2:
-            raise ParseError("Unexpected end of file while parsing")
-
         value = _parse(tokens)
         array.append(value)
 
         token = tokens.popleft()
-        if token not in ',]':
-            raise ParseError(f"Expected ',' or ']', found {token}")
+        if token.type not in ('comma', 'right_bracket'):
+            raise ParseError(
+                f"Expected ',' or ']', found {token.value} "
+                f"(line {token.line} column {token.column})")
 
-        if token == ']':
+        if token.type == 'right_bracket':
             break
 
         # trailing comma check
         if len(tokens) == 0:
-            raise ParseError("Unexpected end of file while parsing")
+            column_end = token.column + len(token.value)
+            raise ParseError(
+                "Unexpected end of file while parsing "
+                f"(line {token.line} column {column_end})")
 
-        if tokens[0] == ']':
-            raise ParseError("Expected value after comma, found ]")
+        if tokens[0].type == 'right_bracket':
+            token = tokens[0]
+            raise ParseError(
+                "Expected value after comma, found ] "
+                f"(line {token.line} column {token.column})")
 
     return array
 
 
-def parse_string(token: str) -> str:
+def parse_string(token: Token) -> str:
     """Parses a string out of a JSON token"""
     chars: List[str] = []
 
     index = 1
-    end = len(token) - 1
+    end = len(token.value) - 1
     while index < end:
-        char = token[index]
+        char = token.value[index]
 
         if char != '\\':
             chars.append(char)
             index += 1
             continue
 
-        next_char = token[index+1]
+        next_char = token.value[index+1]
         if next_char == 'u':
-            hex_string = token[index+2:index+6]
+            hex_string = token.value[index+2:index+6]
             try:
                 unicode_char = literal_eval(f'"\\u{hex_string}"')
             except ValueError as err:
-                raise ParseError(f"Invalid escape: \\u{hex_string}") from err
+                raise ParseError(
+                    f"Invalid escape: \\u{hex_string} "
+                    f"(line {token.line} column {token.column})") from err
 
             chars.append(unicode_char)
             index += 6
@@ -140,7 +167,9 @@ def parse_string(token: str) -> str:
         elif next_char == 't':
             chars.append('\t')
         else:
-            raise ParseError(f"Unknown escape sequence found in {token}")
+            raise ParseError(
+                f"Unknown escape sequence found in {token.value} "
+                f"(line {token.line} column {token.column})")
 
         index += 2
 
@@ -148,33 +177,35 @@ def parse_string(token: str) -> str:
     return string
 
 
-def parse_number(token: str) -> JSONNumber:
+def parse_number(token: Token) -> JSONNumber:
     """Parses a number out of a JSON token"""
     try:
-        if token.isdigit():
-            number: JSONNumber = int(token)
+        if token.value.isdigit():
+            number: JSONNumber = int(token.value)
         else:
-            number = float(token)
+            number = float(token.value)
         return number
 
     except ValueError as err:
-        raise ParseError(f"Invalid token: {token}") from err
+        raise ParseError(
+            f"Invalid token: {token.value} "
+            f"(line {token.line} column {token.column})") from err
 
 
-def _parse(tokens: Deque[str]) -> object:
+def _parse(tokens: Deque[Token]) -> object:
     """Recursive JSON parse implementation"""
     token = tokens.popleft()
 
-    if token == '[':
+    if token.type == 'left_bracket':
         return parse_array(tokens)
 
-    if token == '{':
+    if token.type == 'left_brace':
         return parse_object(tokens)
 
-    if token.startswith('"'):
+    if token.type == 'string':
         return parse_string(token)
 
-    if token[0] == '-' or token[0].isdigit():
+    if token.type == 'number':
         return parse_number(token)
 
     special_tokens = {
@@ -182,10 +213,12 @@ def _parse(tokens: Deque[str]) -> object:
         'false': False,
         'null': None,
     }
-    if token in special_tokens:
-        return special_tokens[token]
+    if token.type in ('boolean', 'null'):
+        return special_tokens[token.value]
 
-    raise ParseError(f"Unexpected token: {token}")
+    raise ParseError(
+        f"Unexpected token: {token.value} "
+        f"(line {token.line} column {token.column})")
 
 
 def parse(json_string: str) -> object:
@@ -194,6 +227,8 @@ def parse(json_string: str) -> object:
 
     value = _parse(tokens)
     if len(tokens) != 0:
-        raise ParseError(f"Invalid JSON at {tokens[0]}")
+        raise ParseError(
+            f"Invalid JSON at {tokens[0].value} "
+            f"(line {tokens[0].line} column {tokens[0].column})")
 
     return value


Test patch:
diff --git a/tests/json_lexer_test.py b/tests/json_lexer_test.py
index d77c1b7..b2199cd 100644
--- a/tests/json_lexer_test.py
+++ b/tests/json_lexer_test.py
@@ -3,8 +3,7 @@ from typing import List
 
 import pytest
 
-import json_parser.lexer
-from json_parser.lexer import TokenizeError
+from json_parser.lexer import tokenize, TokenizeError
 
 
 @pytest.mark.parametrize(
@@ -29,23 +28,32 @@ from json_parser.lexer import TokenizeError
 )
 def test_lexer(json_string: str, expected: List[str]) -> None:
     """JSON lexer tests"""
-    assert list(json_parser.lexer.tokenize(json_string)) == expected
+    assert [token.value for token in tokenize(json_string)] == expected
 
 
 @pytest.mark.parametrize(
     ('json_string', 'error_message'),
     (
         ('', 'Cannot parse empty string'),
-        ('blabla', 'Unknown token found: blabla'),
-        ('"abc', 'Expected end of string'),
-        ('"abc\\"', 'Expected end of string'),
-        ('["a", "b", c]', 'Unknown token found: c'),
+        ('blabla', 'Unknown token found: blabla (line 1 column 1)'),
+        ('"abc', 'Expected end of string (line 1 column 5)'),
+        ('"abc\\"', 'Expected end of string (line 1 column 7)'),
+        ('["a", "b", c]', 'Unknown token found: c (line 1 column 12)'),
+        ('''{
+            "values": ["a", "b", c]
+        }''', 'Unknown token found: c (line 2 column 34)'),
+        ('''{
+            "values": ["a", "b", {
+                "test": "ok",
+                "wow": ["Such", "tests,]
+            }]
+        }''', 'Expected end of string (line 6 column 10)'),
     )
 )
 def test_lexer_failure(json_string: str, error_message: str) -> None:
     """JSON lexer test failutes"""
     with pytest.raises(TokenizeError) as exinfo:
-        json_parser.lexer.tokenize(json_string)
+        tokenize(json_string)
 
     msg, = exinfo.value.args
     assert msg == error_message
diff --git a/tests/json_parser_test.py b/tests/json_parser_test.py
index 533ede4..496625c 100644
--- a/tests/json_parser_test.py
+++ b/tests/json_parser_test.py
@@ -127,13 +127,83 @@ def test_parser(json_string: str, expected: Dict[str, object]) -> None:
 @pytest.mark.parametrize(
     ('json_string', 'error_message'),
     (
-        ('[1, 2, 3,]', 'Expected value after comma, found ]'),
-        ('[1, 2 "hello"]', "Expected ',' or ']', found \"hello\""),
-        ('{35: "test"}', "Expected string key for object, found 35"),
-        ('{"abc":}', "Expected value after colon, found }"),
-        ('{"abc":"def",}', "Expected value after comma, found }"),
-        ('{"abc":', "Unexpected end of file while parsing"),
-        ('[2,', "Unexpected end of file while parsing"),
+        ('[1, 2, 3,]', 'Expected value after comma, found ] (line 1 column 10)'),
+        ('[1, 2 "hello"]', "Expected ',' or ']', found \"hello\" (line 1 column 7)"),
+        ('{35: "test"}', "Expected string key for object, found 35 (line 1 column 2)"),
+        ('{"abc":}', "Expected value after colon, found } (line 1 column 8)"),
+        ('{"abc":"def",}', "Expected value after comma, found } (line 1 column 14)"),
+        ('{"abc":', "Unexpected end of file while parsing (line 1 column 8)"),
+        ('[2,', "Unexpected end of file while parsing (line 1 column 4)"),
+        (
+            """{
+            "results": [
+                {
+                "gender": "male",
+                "name": {
+                    "title": "Mr",
+                    "first": "\u0633\u06cc\u0646\u0627",
+                    "last": "\u0645\u0648\u0633\u0648\u06cc"
+                },
+                "location": {
+                    "street": {
+                      "number": 8134,
+                      "name": "\u0645\u06cc\u062f\u0627\u0646 \u0627\u0645\u0627\u0645 \u062d\u0633\u06cc\u0646"
+                    },
+                    "city": "\u0627\u0631\u0648\u0645\u06cc\u0647",
+                    "state": "\u062e\u0648\u0632\u0633\u062a\u0627\u0646",
+                    "country": "Iran",
+                    "postcode": 24340,
+                    "coordinates": {
+                      "latitude": "27.3083",
+                      "longitude": "-104.2564"
+                    },
+                    "timezone": {
+                    "offset": "0:00",
+                    "description": "Western Europe Time, \n\n    London, Lisbon, Casablanca"
+                    }
+                },
+                "email": "syn.mwswy@example.com",
+                "login": {
+                    "uuid": "8a6da152-019a-40b4-80b0-bfafd5281fd7",
+                    "username": "sadbear764",
+                    "password": "1947",
+                    "salt": "ddKNbUrc",
+                    "md5": "7ff0c750f9b8d7690d50385754a7fe25",
+                    "sha1": "e140544e222f27a2d0aa809ccb00a1d1ca1fda60",
+                    "sha256": "fd5dbd61da24a82f48971a6d027400a2fd5b0808fd108764f17d87aaa61774d9"
+                },
+                "dob": {
+                    "date": "1996-12-06T21:55:10.574Z",
+                    "age": 24
+                },
+                "registered": {
+                    "date": "2013-04-07T05:56:17.049Z",
+                    "age": 7
+                },
+                "phone": "083-15098477",
+                "cell": "0998-569-1505",
+                14: {
+                    "name": "",
+                    "value": null
+                },
+                "picture": {
+                    "large": "https://randomuser.me/api/portraits/men/94.jpg",
+                    "medium": "https://randomuser.me/api/portraits/med/men/94.jpg",
+                    "thumbnail": "https://randomuser.me/api/portraits/thumb/men/94.jpg"
+                },
+                "nat": "IR"
+                }
+            ],
+            "info": {
+                "seed": "db5d8d673b395e5a",
+                "results": 1,
+                "page": 1,
+                "version": "1.3"
+            }
+            }
+            """,
+            "Expected string key for object, found 14 (line 50 column 17)",
+        )
     )
 )
 def test_parser_failure(json_string: str, error_message: str) -> None: