Repository: swaroopch/edn_format

Issue:
not handling discard as expected
`[x #_ y z]` should yield `[Symbol(x), Symbol(z)]` but instead it is failing saying: "Don't know how to handle tag _"


Discussion:
bfontaine: Note this doesn’t work with top-level `#_` usage:
```clojure
;; fails
foo #_ bar

;; works
[foo #_ bar]
```

We may want to fix this later.
swaroopch: @bfontaine `Note this doesn’t work with top-level #_ usage` - is this valid syntax though?
bfontaine: > `Note this doesn’t work with top-level #_ usage` - is this valid syntax though?

The spec is not clear whether we can or not, but `clojure.edn` accepts it. It also accepts empty files even if we don’t know if [it’s valid EDN](https://github.com/edn-format/edn/issues/5).

There are three basic cases I thought about:
1. `#_ value` → raise an EOF error
2. `#_ value1 value2` → parse as `value2`
3. `value1 #_ value2` → parse as `value1`

I haven’t tested it yet but I guess changing the grammar like this should work:
```bnf
root_expression = expression
                | discarded_expression
                | discarded_expressions expression
                | expression discarded_expressions
```

Patch:
diff --git a/edn_format/edn_lex.py b/edn_format/edn_lex.py
index ac0a3af..fc2026e 100644
--- a/edn_format/edn_lex.py
+++ b/edn_format/edn_lex.py
@@ -102,7 +102,8 @@ tokens = ('WHITESPACE',
           'MAP_START',
           'SET_START',
           'MAP_OR_SET_END',
-          'TAG')
+          'TAG',
+          'DISCARD_TAG')
 
 PARTS = {}
 PARTS["non_nums"] = r"\w.*+!\-_?$%&=:#<>@"
@@ -138,7 +139,7 @@ KEYWORD = (":"
            "[{all}]+"
            ")").format(**PARTS)
 TAG = (r"\#"
-       r"\w"
+       r"[a-zA-Z]"  # https://github.com/edn-format/edn/issues/30#issuecomment-8540641
        "("
        "[{all}]*"
        r"\/"
@@ -147,6 +148,8 @@ TAG = (r"\#"
        "[{all}]*"
        ")").format(**PARTS)
 
+DISCARD_TAG = r"\#\_"
+
 t_VECTOR_START = r'\['
 t_VECTOR_END = r'\]'
 t_LIST_START = r'\('
@@ -228,9 +231,10 @@ def t_COMMENT(t):
     pass  # ignore
 
 
-def t_DISCARD(t):
-    r'\#_\S+\b'
-    pass  # ignore
+@ply.lex.TOKEN(DISCARD_TAG)
+def t_DISCARD_TAG(t):
+    t.value = t.value[1:]
+    return t
 
 
 @ply.lex.TOKEN(TAG)
diff --git a/edn_format/edn_parse.py b/edn_format/edn_parse.py
index c2be09d..329584e 100644
--- a/edn_format/edn_parse.py
+++ b/edn_format/edn_parse.py
@@ -56,41 +56,21 @@ def p_term_leaf(p):
     p[0] = p[1]
 
 
-def p_empty_vector(p):
-    """vector : VECTOR_START VECTOR_END"""
-    p[0] = ImmutableList([])
-
-
 def p_vector(p):
     """vector : VECTOR_START expressions VECTOR_END"""
     p[0] = ImmutableList(p[2])
 
 
-def p_empty_list(p):
-    """list : LIST_START LIST_END"""
-    p[0] = tuple()
-
-
 def p_list(p):
     """list : LIST_START expressions LIST_END"""
     p[0] = tuple(p[2])
 
 
-def p_empty_set(p):
-    """set : SET_START MAP_OR_SET_END"""
-    p[0] = frozenset()
-
-
 def p_set(p):
     """set : SET_START expressions MAP_OR_SET_END"""
     p[0] = frozenset(p[2])
 
 
-def p_empty_map(p):
-    """map : MAP_START MAP_OR_SET_END"""
-    p[0] = ImmutableDict({})
-
-
 def p_map(p):
     """map : MAP_START expressions MAP_OR_SET_END"""
     terms = p[2]
@@ -100,14 +80,20 @@ def p_map(p):
     p[0] = ImmutableDict(dict([terms[i:i + 2] for i in range(0, len(terms), 2)]))
 
 
-def p_expressions_expressions_expression(p):
-    """expressions : expressions expression"""
-    p[0] = p[1] + [p[2]]
+def p_discarded_expressions(p):
+    """discarded_expressions : DISCARD_TAG expression discarded_expressions
+                             |"""
+    p[0] = []
+
 
+def p_expressions_expression_expressions(p):
+    """expressions : expression expressions"""
+    p[0] = [p[1]] + p[2]
 
-def p_expressions_expression(p):
-    """expressions : expression"""
-    p[0] = [p[1]]
+
+def p_expressions_empty(p):
+    """expressions : discarded_expressions"""
+    p[0] = []
 
 
 def p_expression(p):
@@ -119,6 +105,11 @@ def p_expression(p):
     p[0] = p[1]
 
 
+def p_expression_discard_expression_expression(p):
+    """expression : DISCARD_TAG expression expression"""
+    p[0] = p[3]
+
+
 def p_expression_tagged_element(p):
     """expression : TAG expression"""
     tag = p[1]
@@ -144,9 +135,13 @@ def p_expression_tagged_element(p):
     p[0] = output
 
 
+def eof():
+    raise EDNDecodeError('EOF Reached')
+
+
 def p_error(p):
     if p is None:
-        raise EDNDecodeError('EOF Reached')
+        eof()
     else:
         raise EDNDecodeError(p)
 


Test patch:
diff --git a/tests.py b/tests.py
index 8f7fcc1..29562d5 100644
--- a/tests.py
+++ b/tests.py
@@ -133,6 +133,12 @@ class EdnTest(unittest.TestCase):
     def check_roundtrip(self, data_input, **kw):
         self.assertEqual(data_input, loads(dumps(data_input, **kw)))
 
+    def check_eof(self, data_input, **kw):
+        with self.assertRaises(EDNDecodeError) as ctx:
+            loads(data_input, **kw)
+
+        self.assertEqual('EOF Reached', str(ctx.exception))
+
     def test_dump(self):
         self.check_roundtrip({1, 2, 3})
         self.check_roundtrip({1, 2, 3}, sort_sets=True)
@@ -339,6 +345,57 @@ class EdnTest(unittest.TestCase):
                              set(seq),
                              sort_sets=True)
 
+    def test_discard(self):
+        for expected, edn_data in (
+            ('[x]', '[x #_ z]'),
+            ('[z]', '[#_ x z]'),
+            ('[x z]', '[x #_ y z]'),
+            ('{1 4}', '{1 #_ 2 #_ 3 4}'),
+            ('[1 2]', '[1 #_ [ #_ [ #_ [ #_ [ #_ 42 ] ] ] ] 2 ]'),
+            ('[1 2 11]', '[1 2 #_ #_ #_ #_ 4 5 6 #_ 7 #_ #_ 8 9 10 11]'),
+            ('()', '(#_(((((((1))))))))'),
+            ('[6]', '[#_ #_ #_ #_ #_ 1 2 3 4 5 6]'),
+            ('[4]', '[#_ #_ 1 #_ 2 3 4]'),
+            ('{:a 1}', '{:a #_:b 1}'),
+            ('[42]', '[42 #_ {:a [1 2 3 4] true false 1 #inst "2017"}]'),
+            ('#{1}', '#{1 #_foo}'),
+            ('"#_ foo"', '"#_ foo"'),
+            ('["#" _]', '[\#_]'),
+            ('[_]', '[#_\#_]'),
+            ('[1]', '[1 #_\n\n42]'),
+            ('{}', '{#_ 1}'),
+        ):
+            self.assertEqual(expected, dumps(loads(edn_data)), edn_data)
+
+    def test_discard_syntax_errors(self):
+        for edn_data in ('#_', '#_ #_ 1', '#inst #_ 2017', '[#_]'):
+            with self.assertRaises(EDNDecodeError):
+                loads(edn_data)
+
+    def test_discard_all(self):
+        for edn_data in (
+            '42', '-1', 'nil', 'true', 'false', '"foo"', '\\space', '\\a',
+            ':foo', ':foo/bar', '[]', '{}', '#{}', '()', '(a)', '(a b)',
+            '[a [[[b] c]] 2]', '#inst "2017"',
+        ):
+            self.assertEqual([1], loads('[1 #_ {}]'.format(edn_data)), edn_data)
+            self.assertEqual([1], loads('[#_ {} 1]'.format(edn_data)), edn_data)
+
+            self.check_eof('#_ {}'.format(edn_data))
+
+            for coll in ('[%s]', '(%s)', '{%s}', '#{%s}'):
+                expected = coll % ""
+                edn_data = coll % '#_ {}'.format(edn_data)
+                self.assertEqual(expected, dumps(loads(edn_data)), edn_data)
+
+    def test_chained_discards(self):
+        for expected, edn_data in (
+            ('[]', '[#_ 1 #_ 2 #_ 3]'),
+            ('[]', '[#_ #_ 1 2 #_ 3]'),
+            ('[]', '[#_ #_ #_ 1 2 3]'),
+        ):
+            self.assertEqual(expected, dumps(loads(edn_data)), edn_data)
+
 
 class EdnInstanceTest(unittest.TestCase):
     def test_hashing(self):