Repository: vertexproject/synapse

Issue:
cortex - lift by inet:cidr gives unexpected results
Lifting nodes by inet:cidr does not appear to behave as expected.

Make a cortex with a a few completely populated pair of /24 blocks
```
core = synapse.cortex.openurl('sqlite:///./cidr.db')
for i in range(0, 256):
    r = core.formTufoByProp('inet:ipv4', "192.168.1.{}".format(i))
    r = core.formTufoByProp('inet:ipv4', "192.168.2.{}".format(i))
    r = core.formTufoByProp('inet:ipv4', "192.168.200.{}".format(i))
```

Then, connect to the cortex using commander and do some lifts:

```
python -m synapse.cortex sqlite:///./cidr.db

# Ensure we have nodes
cli> ask inet:ipv4=192.168.1.0
inet:ipv4 = 192.168.1.0
(1 results)
cli> ask inet:ipv4=192.168.1.200
inet:ipv4 = 192.168.1.200
(1 results)
cli> ask inet:ipv4=192.168.2.255
inet:ipv4 = 192.168.2.255
(1 results)
cli> ask inet:ipv4=192.168.200.100
inet:ipv4 = 192.168.200.100
(1 results)

# Now lift things using the cidr helper - this should yield nodes from 192.168.1.0 -> 192.168.1.255
cli> ask inet:ipv4*inet:cidr=192.168.1.0/24
(0 results)

# However, asking for all nodes in 192.168.2.0/24 does work
cli> ask inet:ipv4*inet:cidr=192.168.2.0/24
inet:ipv4 = 192.168.2.0
inet:ipv4 = 192.168.2.1
inet:ipv4 = 192.168.2.2
...
inet:ipv4 = 192.168.2.253
inet:ipv4 = 192.168.2.254
inet:ipv4 = 192.168.2.255
(256 results)

# As does asking for the 192.168.200/24
cli> ask inet:ipv4*inet:cidr=192.168.200.0/24
inet:ipv4 = 192.168.200.0
inet:ipv4 = 192.168.200.1
inet:ipv4 = 192.168.200.2
...
inet:ipv4 = 192.168.200.253
inet:ipv4 = 192.168.200.254
inet:ipv4 = 192.168.200.255
(256 results)
```

Cross checking with a few online CIDR calculators shows that the 192.168.1.xxx IP nodes would be in the IP range of 192.168.1.0/24

First reported by @therealsilence

@invisig0th is this a bug or a design limitation?

Discussion:

Patch:
diff --git a/synapse/cores/common.py b/synapse/cores/common.py
index 1598c0aa1..070cd0c4e 100644
--- a/synapse/cores/common.py
+++ b/synapse/cores/common.py
@@ -2671,13 +2671,8 @@ class Cortex(EventBus, DataModel, Runtime, Configable, s_ingest.IngestApi):
         return ret
 
     def _tufosByInetCidr(self, prop, valu, limit=None):
-
-        ipv4str, cidr = valu.split('/', 1)
-        ipv4addr, _ = s_datamodel.getTypeParse('inet:ipv4', ipv4str)
-        mask = (2 ** (32 - int(cidr)))
-        ipv4addr &= ~mask
-
-        return self.getTufosBy('range', prop, (ipv4addr, ipv4addr + mask), limit=limit)
+        lowerbound, upperbound = self.getTypeCast('inet:ipv4:cidr', valu)
+        return self.getTufosBy('range', prop, (lowerbound, upperbound), limit=limit)
 
     def _onTufoAddSynType(self, mesg):
         tufo = mesg[1].get('node')
diff --git a/synapse/models/inet.py b/synapse/models/inet.py
index 9630c6e95..b3ae8faa7 100644
--- a/synapse/models/inet.py
+++ b/synapse/models/inet.py
@@ -27,10 +27,18 @@ def ipv4int(valu):
         raise BadTypeValu(valu=valu, type='inet:ipv4', mesg=str(e))
 
 masks = [(0xffffffff - (2 ** (32 - i) - 1)) for i in range(33)]
+cidrmasks = [((0xffffffff - (2 ** (32 - i) - 1)), (2 ** (32 - i))) for i in range(33)]
 
 def ipv4mask(ipv4, mask):
     return ipv4 & masks[mask]
 
+def ipv4cidr(valu):
+    _ipv4str, cidr = valu.split('/', 1)
+    _ipv4addr = ipv4int(_ipv4str)
+    mask = cidrmasks[int(cidr)]
+    lowerbound = _ipv4addr & mask[0]
+    return lowerbound, lowerbound + mask[1]
+
 class IPv4Type(DataType):
     def norm(self, valu, oldval=None):
         if s_compat.isstr(valu):
@@ -287,6 +295,7 @@ class InetMod(CoreModule):
     def initCoreModule(self):
         # add an inet:defang cast to swap [.] to .
         self.core.addTypeCast('inet:defang', castInetDeFang)
+        self.core.addTypeCast('inet:ipv4:cidr', ipv4cidr)
         self.onFormNode('inet:fqdn', self.onTufoFormFqdn)
         self.onFormNode('inet:passwd', self.onTufoFormPasswd)
         self.revCoreModl()
@@ -332,7 +341,7 @@ class InetMod(CoreModule):
                     'inet:cidr4',
                     {'ctor': 'synapse.models.inet.CidrType', 'doc': 'An IPv4 CIDR type', 'ex': '1.2.3.0/24'}),
 
-                ('inet:urlfile', {'subof': 'comp', 'fields':'url=inet:url,file=file:bytes',
+                ('inet:urlfile', {'subof': 'comp', 'fields': 'url=inet:url,file=file:bytes',
                                   'doc': 'A File at a Universal Resource Locator (URL)'}),
                 ('inet:net4',
                  {'subof': 'sepr', 'sep': '-', 'fields': 'min,inet:ipv4|max,inet:ipv4', 'doc': 'An IPv4 address range',


Test patch:
diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py
index de94ab278..4255a95f0 100644
--- a/synapse/tests/test_cortex.py
+++ b/synapse/tests/test_cortex.py
@@ -623,14 +623,74 @@ class CortexTest(SynTest):
             ipint, _ = tlib.getTypeParse('inet:ipv4', ip)
             ipc = core.formTufoByProp('inet:ipv4', ipint)
 
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.4/32')), 1)
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.4/31')), 2)
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.1/30')), 4)
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.2/30')), 4)
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.1/29')), 8)
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.1/28')), 16)
-
-        self.eq(len(core.getTufosBy('inet:cidr', 'inet:ipv4', '192.168.0.0/16')), 2)
+        # Validate the content we get from cidr lookups is correctly bounded
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.4/32')
+        self.eq(len(nodes), 1)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.4')
+
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.4/31')
+        self.eq(len(nodes), 2)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.4')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.5')
+
+        # 10.2.1.1/30 is 10.2.1.0 -> 10.2.1.3 but we don't have 10.2.1.0 in the core
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.1/30')
+        self.eq(len(nodes), 3)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.1')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.3')
+
+        # 10.2.1.2/30 is 10.2.1.0 -> 10.2.1.3 but we don't have 10.2.1.0 in the core
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.2/30')
+        self.eq(len(nodes), 3)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.1')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.3')
+
+        # 10.2.1.1/29 is 10.2.1.0 -> 10.2.1.7 but we don't have 10.2.1.0 in the core
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.1/29')
+        self.eq(len(nodes), 7)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.1')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.7')
+
+        # 10.2.1.8/29 is 10.2.1.8 -> 10.2.1.15
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.8/29')
+        self.eq(len(nodes), 8)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.8')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.15')
+
+        # 10.2.1.1/28 is 10.2.1.0 -> 10.2.1.15 but we don't have 10.2.1.0 in the core
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '10.2.1.1/28')
+        self.eq(len(nodes), 15)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.1')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '10.2.1.15')
+
+        # 192.168.0.0/16 is 192.168.0.0 -> 192.168.255.255 but we only have two nodes in this range
+        nodes = core.getTufosBy('inet:cidr', 'inet:ipv4', '192.168.0.0/16')
+        self.eq(len(nodes), 2)
+        nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+        self.eq(test_repr, '192.168.0.1')
+        test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+        self.eq(test_repr, '192.168.255.254')
 
     def test_cortex_tufo_by_postgres(self):
 
@@ -2227,3 +2287,75 @@ class CortexTest(SynTest):
                 self.eq(node[1].get('foo:bar:duck'), 'mallard')
                 node2 = core.formTufoByProp('foo:bar', 'I am a robot', duck='mandarin')
                 self.eq(node2[1].get('foo:bar:duck'), 'mandarin')
+
+    def test_cortex_lift_by_cidr(self):
+
+        with s_cortex.openurl('ram:///') as core:
+            # Add a bunch of nodes
+            for n in range(0, 256):
+                r = core.formTufoByProp('inet:ipv4', '192.168.1.{}'.format(n))
+                r = core.formTufoByProp('inet:ipv4', '192.168.2.{}'.format(n))
+                r = core.formTufoByProp('inet:ipv4', '192.168.200.{}'.format(n))
+
+            # Confirm we have nodes
+            self.eq(len(core.eval('inet:ipv4="192.168.1.0"')), 1)
+            self.eq(len(core.eval('inet:ipv4="192.168.1.255"')), 1)
+            self.eq(len(core.eval('inet:ipv4="192.168.2.0"')), 1)
+            self.eq(len(core.eval('inet:ipv4="192.168.2.255"')), 1)
+            self.eq(len(core.eval('inet:ipv4="192.168.200.0"')), 1)
+
+            # Do cidr lifts
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.2.0/24')
+            nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+            self.eq(len(nodes), 256)
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[10][1].get('inet:ipv4'))
+            self.eq(test_repr, '192.168.2.10')
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+            self.eq(test_repr, '192.168.2.0')
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+            self.eq(test_repr, '192.168.2.255')
+
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.200.0/24')
+            self.eq(len(nodes), 256)
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[10][1].get('inet:ipv4'))
+            self.true(test_repr.startswith('192.168.200.'))
+
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.1.0/24')
+            self.eq(len(nodes), 256)
+            nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[10][1].get('inet:ipv4'))
+            self.eq(test_repr, '192.168.1.10')
+
+            # Try a complicated /24
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.1.1/24')
+            self.eq(len(nodes), 256)
+            nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+            self.eq(test_repr, '192.168.1.0')
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[255][1].get('inet:ipv4'))
+            self.eq(test_repr, '192.168.1.255')
+
+            # Try a /23
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.0.0/23')
+            self.eq(len(nodes), 256)
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[10][1].get('inet:ipv4'))
+            self.true(test_repr.startswith('192.168.1.'))
+
+            # Try a /25
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.1.0/25')
+            nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+            self.eq(len(nodes), 128)
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[-1][1].get('inet:ipv4'))
+            self.true(test_repr.startswith('192.168.1.127'))
+
+            # Try a /25
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.1.128/25')
+            nodes.sort(key=lambda x: x[1].get('inet:ipv4'))
+            self.eq(len(nodes), 128)
+            test_repr = core.getTypeRepr('inet:ipv4', nodes[0][1].get('inet:ipv4'))
+            self.true(test_repr.startswith('192.168.1.128'))
+
+            # Try a /16
+            nodes = core.eval('inet:ipv4*inet:cidr=192.168.0.0/16')
+            self.eq(len(nodes), 256 * 3)