Repository: secdev/scapy

Issue:
[Feature Request] Extensible Record Format (ERF) Support
<!--
Things to consider

1.  Please check that you are using the latest scapy version, e.g. installed via:

    `pip install --upgrade git+git://github.com/secdev/scapy`

2.  If you are here to ask a question - please check previous issues and online resources, and consider using gitter instead: <https://gitter.im/secdev/scapy>

3.  Please understand that **this is not a forum** but an issue tracker. The following article explains why you should limit questions asked on Github issues: <https://medium.com/@methane/why-you-must-not-ask-questions-on-github-issues-51d741d83fde>
-->

#### Brief description

<!-- describe the main issue in one sentence -->
<!-- if possible, describe what components / protocols could be affected by the issue (e.g. wrpcap() + IPv6, it is likely this also affects XXX) -->

Me and my colleague would like to request for Extensible Record Format (ERF) support for Scapy. Specifically, the second type, TYPE_ETH, which is used for the ethernet captures. 
Previously when we want to read an ERF capture file, we convert the ERF file to pcap using wireshark or tshark and pass the pcap to scapy. We think it would be nice if there is direct support for this type is available in Scapy.
Do you know whether this feature already existed previously or has anyone requested for it? We hope we could help contribute to supporting the read and write of the ERF packet.

We would like to hear what do you think about this!
<!-- if needed - further information to get a picture of your setup (e.g. a sketch of your network setup) -->

#### Related resources

<!-- traces / sample pcaps (stripped to the relevant frames), related standards, RFCs or other resources -->
https://www.endace.com/erf-extensible-record-format-types.pdf



Discussion:
Thanks for your message. This sounds like a good idea.

Nobody requested this future before. Feel free to contribute and submit a PR.

Resolving patch:
diff --git a/scapy/utils.py b/scapy/utils.py
--- a/scapy/utils.py
+++ b/scapy/utils.py
@@ -1701,7 +1701,7 @@ def recv(self, size=MTU):
         return self.read_packet()
 
 
-class RawPcapWriter:
+class RawPcapWriter(object):
     """A stream PCAP writer with more control than wrpcap()"""
 
     def __init__(self,
@@ -1978,6 +1978,219 @@ def write_packet(self,
         )
 
 
+@conf.commands.register
+def rderf(filename, count=-1):
+    # type: (Union[IO[bytes], str], int) -> PacketList
+    """Read a ERF file and return a packet list
+
+    :param count: read only <count> packets
+    """
+    with ERFEthernetReader(filename) as fdesc:
+        return fdesc.read_all(count=count)
+
+
+class ERFEthernetReader_metaclass(PcapReader_metaclass):
+    def __call__(cls, filename):  # type: ignore
+        # type: (Union[IO[bytes], str]) -> Any
+        i = cls.__new__(cls, cls.__name__, cls.__bases__, cls.__dict__)
+        filename, fdesc = cls.open(filename)
+        try:
+            i.__init__(filename, fdesc)
+            return i
+        except (Scapy_Exception, EOFError):
+            pass
+
+        if "alternative" in cls.__dict__:
+            cls = cls.__dict__["alternative"]
+            i = cls.__new__(cls, cls.__name__, cls.__bases__, cls.__dict__)
+            try:
+                i.__init__(filename, fdesc)
+                return i
+            except (Scapy_Exception, EOFError):
+                pass
+
+        raise Scapy_Exception("Not a supported capture file")
+
+    @staticmethod
+    def open(fname  # type: ignore
+             ):
+        # type: (...) -> Tuple[str, _ByteStream]
+        """Open (if necessary) filename"""
+        if isinstance(fname, str):
+            filename = fname
+            try:
+                with gzip.open(filename, "rb") as tmp:
+                    tmp.read(1)
+                fdesc = gzip.open(filename, "rb")  # type: _ByteStream
+            except IOError:
+                fdesc = open(filename, "rb")
+
+        else:
+            fdesc = fname
+            filename = getattr(fdesc, "name", "No name")
+        return filename, fdesc
+
+
+@six.add_metaclass(ERFEthernetReader_metaclass)
+class ERFEthernetReader(PcapReader):
+
+    def __init__(self, filename, fdesc=None):  # type: ignore
+        # type: (Union[IO[bytes], str], IO[bytes]) -> None
+        self.filename = filename  # type: ignore
+        self.f = fdesc
+        self.power = Decimal(10) ** Decimal(-9)
+
+    # time is in 64-bits Endace's format which can be see here:
+    # https://www.endace.com/erf-extensible-record-format-types.pdf
+    def _convert_erf_timestamp(self, t):
+        # type: (int) -> EDecimal
+        sec = t >> 32
+        frac_sec = t & 0xffffffff
+        frac_sec *= 10**9
+        frac_sec += (frac_sec & 0x80000000) << 1
+        frac_sec >>= 32
+        return EDecimal(sec + self.power * frac_sec)
+
+    # The details of ERF Packet format can be see here:
+    # https://www.endace.com/erf-extensible-record-format-types.pdf
+    def read_packet(self, size=MTU):
+        # type: (int) -> Packet
+
+        # General ERF Header have exactly 16 bytes
+        hdr = self.f.read(16)
+        if len(hdr) < 16:
+            raise EOFError
+
+        # The timestamp is in little-endian byte-order.
+        time = struct.unpack('<Q', hdr[:8])[0]
+        # The rest is in big-endian byte-order.
+        # Ignoring flags and lctr (loss counter) since they are ERF specific
+        # header fields which Packet object does not support.
+        type, _, rlen, _, wlen = struct.unpack('>BBHHH', hdr[8:])
+        # Check if the type != 0x02, type Ethernet
+        if type & 0x02 == 0:
+            raise Scapy_Exception("Invalid ERF Type (Not TYPE_ETH)")
+
+        # If there are extended headers, ignore it because Packet object does
+        # not support it. Extended headers size is 8 bytes before the payload.
+        if type & 0x80:
+            _ = self.f.read(8)
+            s = self.f.read(rlen - 24)
+        else:
+            s = self.f.read(rlen - 16)
+
+        # Ethernet has 2 bytes of padding containing `offset` and `pad`. Both
+        # of the fields are disregarded by Endace.
+        p = s[2:size]
+        from scapy.layers.l2 import Ether
+        try:
+            p = Ether(p)
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            if conf.debug_dissector:
+                from scapy.sendrecv import debug
+                debug.crashed_on = (Ether, s)
+                raise
+            if conf.raw_layer is None:
+                # conf.raw_layer is set on import
+                import scapy.packet  # noqa: F401
+            p = conf.raw_layer(s)
+
+        p.time = self._convert_erf_timestamp(time)
+        p.wirelen = wlen
+
+        return p
+
+
+@conf.commands.register
+def wrerf(filename,  # type: Union[IO[bytes], str]
+          pkt,  # type: _PacketIterable
+          *args,  # type: Any
+          **kargs  # type: Any
+          ):
+    # type: (...) -> None
+    """Write a list of packets to a ERF file
+
+    :param filename: the name of the file to write packets to, or an open,
+        writable file-like object. The file descriptor will be
+        closed at the end of the call, so do not use an object you
+        do not want to close (e.g., running wrerf(sys.stdout, [])
+        in interactive mode will crash Scapy).
+    :param gz: set to 1 to save a gzipped capture
+    :param append: append packets to the capture file instead of
+        truncating it
+    :param sync: do not bufferize writes to the capture file
+    """
+    with ERFEthernetWriter(filename, *args, **kargs) as fdesc:
+        fdesc.write(pkt)
+
+
+class ERFEthernetWriter(PcapWriter):
+    """A stream ERF Ethernet writer with more control than wrerf()"""
+
+    def __init__(self,
+                 filename,  # type: Union[IO[bytes], str]
+                 gz=False,  # type: bool
+                 append=False,  # type: bool
+                 sync=False,  # type: bool
+                 ):
+        # type: (...) -> None
+        """
+        :param filename: the name of the file to write packets to, or an open,
+            writable file-like object.
+        :param gz: compress the capture on the fly
+        :param append: append packets to the capture file instead of
+            truncating it
+        :param sync: do not bufferize writes to the capture file
+        """
+        super(ERFEthernetWriter, self).__init__(filename,
+                                                gz=gz,
+                                                append=append,
+                                                sync=sync)
+
+    def write(self, pkt):  # type: ignore
+        # type: (_PacketIterable) -> None
+        """
+        Writes a Packet, a SndRcvList object, or bytes to a ERF file.
+
+        :param pkt: Packet(s) to write (one record for each Packet)
+        :type pkt: iterable[scapy.packet.Packet], scapy.packet.Packet
+        """
+        # Import here to avoid circular dependency
+        from scapy.supersocket import IterSocket
+        for p in IterSocket(pkt).iter:
+            self.write_packet(p)
+
+    def write_packet(self, pkt):  # type: ignore
+        # type: (Packet) -> None
+
+        if hasattr(pkt, "time"):
+            sec = int(pkt.time)
+            usec = int((int(round((pkt.time - sec) * 10**9)) << 32) / 10**9)
+            t = (sec << 32) + usec
+        else:
+            t = int(time.time()) << 32
+
+        # There are 16 bytes of headers + 2 bytes of padding before the packets
+        # payload.
+        rlen = len(pkt) + 18
+
+        if hasattr(pkt, "wirelen"):
+            wirelen = pkt.wirelen
+        if wirelen is None:
+            wirelen = rlen
+
+        self.f.write(struct.pack("<Q", t))
+        self.f.write(struct.pack(">BBHHHH", 2, 0, rlen, 0, wirelen, 0))
+        self.f.write(bytes(pkt))
+        self.f.flush()
+
+    def close(self):
+        # type: () -> Optional[Any]
+        return self.f.close()
+
+
 @conf.commands.register
 def import_hexcap(input_string=None):
     # type: (Optional[str]) -> bytes


Test patch:
diff --git a/test/regression.uts b/test/regression.uts
--- a/test/regression.uts
+++ b/test/regression.uts
@@ -2363,6 +2363,53 @@ with mock.patch("scapy.utils.warning") as warning:
     os.remove(filename)
     assert any("Inconsistent" in arg for arg in warning.call_args[0])
 
+############
+############
++ ERF Ethernet format support 
+
+= Variable creations
+erffile = BytesIO(b'3;!E_9\x92_\x02\x04\x00p\x00\x00\x00P\x00\x00\x00\x0fS?\xca\xc0\x1cjz\x18\x90\xed\x81\x00\x01:\x08\x00E\x00\x00(\xdf\xab@\x00;\x06\xb3s\n\x01]\xdb\n\xfb9\xda\xc3v\x84\xecD\x16\xb9\xab\xda\xa1b\xf9P\x10f\x98\x18\xcb\x00\x00\x00\x00\x90\x9e\xd7\xd2_\x929_\x0f\x9e\xcd\x1f\x01\x88\xb9\x15[/s<\x01\x88\xb9\x15[/\xcd\x1f\x01\x88\xb9\x15[/0\xcd"E_9\x92_\x02\x04\x00p\x00\x00\x00P\x00\x00\x1cjz\x18\x90\xed\x00\x0fS?\xca\xc0\x08\x00E\x00\x00(\xa2\xdd@\x00@\x06\xebA\n\xfb9\xda\n\x01]\xdb\x84\xec\xc3v\xda\xa1b\xf9D\x16\xb9\xacP\x10\x9a\xf0\xe4q\x00\x00\x00\x00\x00\x00\x00\x00o\xbc\xe2{_\x929_\x0f\x9f+3\x01\x88\xb9\x15u\x1e(^\x01\x88\xb9\x15u\x1e+3\x01\x88\xb9\x15u\x1e')
+erffilewithheader = BytesIO(b'4;!E_9\x92_\x82\x00\x00x\x00\x00\x00P\x00\x00\x1a+<M^o\x00\x00\x00\x0fS?\xca\xc0\x1cjz\x18\x90\xed\x81\x00\x01:\x08\x00E\x00\x00(\xdf\xab@\x00;\x06\xb3s\n\x01]\xdb\n\xfb9\xda\xc3v\x84\xecD\x16\xb9\xab\xda\xa1b\xf9P\x10f\x98\x18\xcb\x00\x00\x00\x00\x90\x9e\xd7\xd2_\x929_\x0f\x9e\xcd\x1f\x01\x88\xb9\x15[/s<\x01\x88\xb9\x15[/\xcd\x1f\x01\x88\xb9\x15[/0\xcd"E_9\x92_\x82\x00\x00x\x00\x00\x00P\x00\x00\x1a+<M^o\x00\x00\x1cjz\x18\x90\xed\x00\x0fS?\xca\xc0\x08\x00E\x00\x00(\xa2\xdd@\x00@\x06\xebA\n\xfb9\xda\n\x01]\xdb\x84\xec\xc3v\xda\xa1b\xf9D\x16\xb9\xacP\x10\x9a\xf0\xe4q\x00\x00\x00\x00\x00\x00\x00\x00o\xbc\xe2{_\x929_\x0f\x9f+3\x01\x88\xb9\x15u\x1e(^\x01\x88\xb9\x15u\x1e+3\x01\x88\xb9\x15u\x1e')
+
+= Check reading of ERF Ethernet file
+pkterf = rderf(erffile)
+assert pkterf[0].time == 1603418463.270038318
+assert pkterf[0][IP].src == "10.1.93.219"
+assert pkterf[0][IP].dst == "10.251.57.218"
+assert pkterf[0][Ether].src == "1c:6a:7a:18:90:ed"
+assert pkterf[0][Ether].dst == "00:0f:53:3f:ca:c0"
+
+= Check writing of ERF Ethernet file
+import os, tempfile
+fdesc, filename = tempfile.mkstemp()
+fdesc = os.fdopen(fdesc, "wb")
+wrerf(fdesc, pkterf)
+fdesc.close()
+newpkterf = rderf(filename)
+
+assert pkterf[1][Ether].src == newpkterf[1][Ether].src
+
+assert len(pkterf) == len(newpkterf)
+assert newpkterf[0].time is not None
+assert newpkterf[0].wirelen is not None
+assert newpkterf[0].time == pkterf[0].time
+assert newpkterf[0].wirelen == pkterf[0].wirelen
+assert newpkterf[1].time is not None
+assert newpkterf[1].wirelen is not None
+assert newpkterf[1].time == pkterf[1].time
+assert newpkterf[1].wirelen == pkterf[1].wirelen
+
+_, filename = tempfile.mkstemp()
+wrerf(filename, pkterf, append=True)
+wrerf(filename, pkterf, append=True)
+newdoublepkterf = rderf(filename)
+
+assert len(newpkterf) * 2 == len(newdoublepkterf)
+
+= Check rderf
+pkterf = rderf(erffilewithheader)
+assert pkterf[1].time == 1603418463.270062279
+assert pkterf[1][Ether].src == "00:0f:53:3f:ca:c0"
 
 ############
 ############