Repository: vertexproject/synapse

Issue:
regex macro filter eats query remainder
seems like string parsing after ~= needs to use the latest function in use by the rest of the macros that can be either whitespace *or* quote bounded.

Discussion:

Patch:
diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py
index 45916ae0b..c2b154683 100644
--- a/synapse/lib/syntax.py
+++ b/synapse/lib/syntax.py
@@ -17,7 +17,7 @@ tagfilt = varset.union({'#', '*', '@'})
 alphaset = set('abcdefghijklmnopqrstuvwxyz')
 
 # this may be used to meh() potentially unquoted values
-valmeh = whites.union({'(',')','=',',','[',']'})
+valmeh = whites.union({'(', ')', '=', ',', '[', ']'})
 
 def nom(txt, off, cset, trim=True):
     '''
@@ -95,15 +95,14 @@ def parse_list(text, off=0, trim=True):
 
         _, off = nom(text, off, whites)
 
-        valu,off = parse_valu(text, off)
+        valu, off = parse_valu(text, off)
 
         _, off = nom(text, off, whites)
 
-
         # check for foo=bar kw tuple syntax
         if nextchar(text, off, '='):
 
-            _, off = nom(text, off+1, whites)
+            _, off = nom(text, off + 1, whites)
 
             vval, off = parse_valu(text, off)
 
@@ -373,7 +372,7 @@ def parse_ques(text, off=0, trim=True):
 
             if textpart.startswith(ctxt):
                 ques['cmp'] = cmpr
-                ques['valu'], off = parse_oarg(text, off + len(ctxt))
+                ques['valu'], off = parse_valu(text, off + len(ctxt))
                 break
 
         break
@@ -384,7 +383,7 @@ def parse_valu(text, off=0):
     '''
     Special syntax for the right side of equals in a macro
     '''
-    _, off = nom(text,off,whites)
+    _, off = nom(text, off, whites)
 
     if nextchar(text, off, '('):
         return parse_list(text, off)


Test patch:
diff --git a/synapse/tests/test_syntax.py b/synapse/tests/test_syntax.py
index bd22f99c2..aa6f4ced3 100644
--- a/synapse/tests/test_syntax.py
+++ b/synapse/tests/test_syntax.py
@@ -66,7 +66,20 @@ class StormSyntaxTest(SynTest):
         insts = s_syntax.parse('+foo:bar~="hehe" -foo:bar~="hoho"')
         self.eq(insts[0], ('filt', {'prop': 'foo:bar', 'mode': 'must', 'cmp': 're', 'valu': 'hehe'}))
         self.eq(insts[1], ('filt', {'prop': 'foo:bar', 'mode': 'cant', 'cmp': 're', 'valu': 'hoho'}))
-        self.eq({'woot': 10}, {'woot': 10})
+        # Real world example with and without quotes around the regex valu
+        insts = s_syntax.parse('#sig.mal.pennywise totags() +syn:tag~="bhv.aka" fromtags()')
+        self.eq(len(insts), 4)
+        self.eq(insts[0], ('alltag', {'args': ('sig.mal.pennywise',), 'kwlist': []}))
+        self.eq(insts[1], ('totags', {'args': [], 'kwlist': []}))
+        self.eq(insts[2], ('filt', {'cmp': 're', 'prop': 'syn:tag', 'mode': 'must', 'valu': 'bhv.aka'}))
+        self.eq(insts[3], ('fromtags', {'args': [], 'kwlist': []}))
+        # And without quotes
+        insts = s_syntax.parse('#sig.mal.pennywise totags() +syn:tag~=bhv.aka fromtags()')
+        self.eq(len(insts), 4)
+        self.eq(insts[0], ('alltag', {'args': ('sig.mal.pennywise',), 'kwlist': []}))
+        self.eq(insts[1], ('totags', {'args': [], 'kwlist': []}))
+        self.eq(insts[2], ('filt', {'cmp': 're', 'prop': 'syn:tag', 'mode': 'must', 'valu': 'bhv.aka'}))
+        self.eq(insts[3], ('fromtags', {'args': [], 'kwlist': []}))
 
     def test_storm_syntax_by(self):
         insts = s_syntax.parse('woot/foo:bar*baz="hehe"')
@@ -85,16 +98,16 @@ class StormSyntaxTest(SynTest):
         self.eq(insts[0], s_syntax.oper('lift', 'inet:fqdn', '1.2.3.4', by='eq'))
 
     def test_storm_syntax_comp_opts(self):
-        valu,off = s_syntax.parse_valu('(foo,bar,baz=faz)')
-        self.eq( valu, ['foo','bar',('baz','faz')] )
+        valu, off = s_syntax.parse_valu('(foo,bar,baz=faz)')
+        self.eq(valu, ['foo', 'bar', ('baz', 'faz')])
 
-        valu,off = s_syntax.parse_valu('(foo, bar, baz=faz)')
-        self.eq( valu, ['foo','bar',('baz','faz')] )
+        valu, off = s_syntax.parse_valu('(foo, bar, baz=faz)')
+        self.eq(valu, ['foo', 'bar', ('baz', 'faz')])
 
-        valu,off = s_syntax.parse_valu('(   foo   ,    bar   ,   baz    =     faz    )')
-        self.eq( valu, ['foo','bar',('baz','faz')] )
+        valu, off = s_syntax.parse_valu('(   foo   ,    bar   ,   baz    =     faz    )')
+        self.eq(valu, ['foo', 'bar', ('baz', 'faz')])
 
     def test_storm_syntax_edit(self):
         inst0 = s_syntax.parse('[inet:ipv4=127.0.0.1 inet:ipv4=127.0.0.2]')
         inst1 = s_syntax.parse('   [   inet:ipv4   =    127.0.0.1  inet:ipv4   =   127.0.0.2   ]   ')
-        self.eq(inst0,inst1)
+        self.eq(inst0, inst1)