Repository: secdev/scapy

Issue:
Nanosecond libpcap
Hi,

I wish I could use scapy for nanosecond analysis, but I get a 
"Not a supported capture file", when trying to read a pcap encoded as Nanosecond lib pcap.
(I'm using the latest revision)

Enclosed the same 3 packets encoded by wireshark with different settings.
the nanosecondlibpcap.pcap file is the one I'd like to be able to read.
We should be able to see : 
packet 1 at 15:21:09:000150304
packet 2 at 15:21:09:000169406
packet 3 at 15:21:09:000216777

Although pcapng seems to be able to use nanoseconds, somehow the nanosecond part is lost (at least when visualizing with wireshark)

[pcaps.zip](https://github.com/secdev/scapy/files/311692/pcaps.zip)



Discussion:
Hi I was trying to use a nanosecond libpcap dataset as well and I am getting a bad magic error. I would really like to use Scapy for this project. 
Thanks.
you can try dpkt which is more lightweight and does the job

Resolving patch:
diff --git a/scapy/utils.py b/scapy/utils.py
--- a/scapy/utils.py
+++ b/scapy/utils.py
@@ -653,10 +653,18 @@ class RawPcapReader:
     def __init__(self, filename, fdesc, magic):
         self.filename = filename
         self.f = fdesc
-        if magic == "\xa1\xb2\xc3\xd4": #big endian
+        if magic == "\xa1\xb2\xc3\xd4": # big endian
             self.endian = ">"
-        elif magic == "\xd4\xc3\xb2\xa1": #little endian
+            self.nano = False
+        elif magic == "\xd4\xc3\xb2\xa1": # little endian
             self.endian = "<"
+            self.nano = False
+        elif magic == "\xa1\xb2\x3c\x4d":  # big endian, nanosecond-precision
+            self.endian = ">"
+            self.nano = True
+        elif magic == "\x4d\x3c\xb2\xa1":  # little endian, nanosecond-precision
+            self.endian = "<"
+            self.nano = True
         else:
             raise Scapy_Exception(
                 "Not a pcap capture file (bad magic: %r)" % magic
@@ -755,7 +763,7 @@ def read_packet(self, size=MTU):
             if conf.debug_dissector:
                 raise
             p = conf.raw_layer(s)
-        p.time = sec+0.000001*usec
+        p.time = sec + (0.000000001 if self.nano else 0.000001) * usec
         return p
     def read_all(self,count=-1):
         res = RawPcapReader.read_all(self, count)
@@ -776,10 +784,12 @@ class RawPcapNgReader(RawPcapReader):
     def __init__(self, filename, fdesc, magic):
         self.filename = filename
         self.f = fdesc
-        # A list of (linktype, snaplen); will be populated by IDBs.
+        # A list of (linktype, snaplen, tsresol); will be populated by IDBs.
         self.interfaces = []
         self.blocktypes = {
             1: self.read_block_idb,
+            2: self.read_block_pkt,
+            3: self.read_block_spb,
             6: self.read_block_epb,
         }
         if magic != "\x0a\x0d\x0d\x0a": # PcapNg:
@@ -809,12 +819,14 @@ def read_packet(self, size=MTU):
             except struct.error:
                 return None
             block = self.f.read(blocklen - 12)
+            if blocklen % 4:
+                pad = self.f.read(4 - (blocklen % 4))
+                warning("PcapNg: bad blocklen %d (MUST be a multiple of 4. "
+                        "Ignored padding %r" % (blocklen, pad))
             try:
                 if (blocklen,) != struct.unpack(self.endian + 'I',
                                                 self.f.read(4)):
-                    raise Scapy_Exception(
-                        "Invalid pcapng block (bad blocklen)"
-                    )
+                    warning("PcapNg: Invalid pcapng block (bad blocklen)")
             except struct.error:
                 return None
             res = self.blocktypes.get(blocktype,
@@ -824,14 +836,57 @@ def read_packet(self, size=MTU):
 
     def read_block_idb(self, block, _):
         """Interface Description Block"""
-        self.interfaces.append(struct.unpack(self.endian + "HxxI", block[:8]))
+        options = block[16:]
+        tsresol = 1000000
+        while len(options) >= 4:
+            code, length = struct.unpack(self.endian + "HH", options[:4])
+            # PCAP Next Generation (pcapng) Capture File Format
+            # 4.2. - Interface Description Block
+            # http://xml2rfc.tools.ietf.org/cgi-bin/xml2rfc.cgi?url=https://raw.githubusercontent.com/pcapng/pcapng/master/draft-tuexen-opsawg-pcapng.xml&modeAsFormat=html/ascii&type=ascii#rfc.section.4.2
+            if code == 9 and length == 1 and len(options) >= 5:
+                tsresol = ord(options[4])
+                tsresol = (2 if tsresol & 128 else 10) ** (tsresol & 127)
+            if code == 0:
+                if length != 0:
+                    warning("PcapNg: invalid option length %d for end-of-option" % length)
+                break
+            if length % 4:
+                length += (4 - (length % 4))
+            options = options[4 + length:]
+        self.interfaces.append(struct.unpack(self.endian + "HxxI", block[:8])
+                               + (tsresol,))
 
     def read_block_epb(self, block, size):
         """Enhanced Packet Block"""
-        intid, sec, usec, caplen, wirelen = struct.unpack(self.endian + "5I",
-                                                          block[:20])
+        intid, tshigh, tslow, caplen, wirelen = struct.unpack(
+            self.endian + "5I",
+            block[:20],
+        )
         return (block[20:20 + caplen][:size],
-                (self.interfaces[intid][0], sec, usec, wirelen))
+                (self.interfaces[intid][0], self.interfaces[intid][2],
+                 tshigh, tslow, wirelen))
+
+    def read_block_spb(self, block, size):
+        """Simple Packet Block"""
+        # "it MUST be assumed that all the Simple Packet Blocks have
+        # been captured on the interface previously specified in the
+        # first Interface Description Block."
+        intid = 0
+        wirelen, = struct.unpack(self.endian + "I", block[:4])
+        caplen = min(wirelen, self.interfaces[intid][1])
+        return (block[4:4 + caplen][:size],
+                (self.interfaces[intid][0], self.interfaces[intid][2],
+                 None, None, wirelen))
+
+    def read_block_pkt(self, block, size):
+        """(Obsolete) Packet Block"""
+        intid, drops, tshigh, tslow, caplen, wirelen = struct.unpack(
+            self.endian + "HH4I",
+            block[:20],
+        )
+        return (block[20:20 + caplen][:size],
+                (self.interfaces[intid][0], self.interfaces[intid][2],
+                 tshigh, tslow, wirelen))
 
 
 class PcapNgReader(RawPcapNgReader):
@@ -845,7 +900,7 @@ def read_packet(self, size=MTU):
         rp = RawPcapNgReader.read_packet(self, size=size)
         if rp is None:
             return None
-        s, (linktype, sec, usec, wirelen) = rp
+        s, (linktype, tsresol, tshigh, tslow, wirelen) = rp
         try:
             p = conf.l2types[linktype](s)
         except KeyboardInterrupt:
@@ -854,7 +909,8 @@ def read_packet(self, size=MTU):
             if conf.debug_dissector:
                 raise
             p = conf.raw_layer(s)
-        p.time = sec+0.000001*usec
+        if tshigh is not None:
+            p.time = float((tshigh << 32) + tslow) / tsresol
         return p
     def read_all(self,count=-1):
         res = RawPcapNgReader.read_all(self, count)
@@ -866,16 +922,18 @@ def recv(self, size=MTU):
 
 class RawPcapWriter:
     """A stream PCAP writer with more control than wrpcap()"""
-    def __init__(self, filename, linktype=None, gz=False, endianness="", append=False, sync=False):
+    def __init__(self, filename, linktype=None, gz=False, endianness="",
+                 append=False, sync=False, nano=False):
         """
-filename: the name of the file to write packets to, or an open,
-          writable file-like object.
-linktype: force linktype to a given value. If None, linktype is taken
-          from the first writer packet
-gz: compress the capture on the fly
+filename:   the name of the file to write packets to, or an open,
+            writable file-like object.
+linktype:   force linktype to a given value. If None, linktype is taken
+            from the first writer packet
+gz:         compress the capture on the fly
 endianness: force an endianness (little:"<", big:">"). Default is native
-append: append packets to the capture file instead of truncating it
-sync: do not bufferize writes to the capture file
+append:     append packets to the capture file instead of truncating it
+sync:       do not bufferize writes to the capture file
+nano:       use nanosecond-precision (requires libpcap >= 1.5.0)
 
         """
         
@@ -885,6 +943,7 @@ def __init__(self, filename, linktype=None, gz=False, endianness="", append=Fals
         self.gz = gz
         self.endian = endianness
         self.sync = sync
+        self.nano = nano
         bufsz=4096
         if sync:
             bufsz = 0
@@ -913,7 +972,7 @@ def _write_header(self, pkt):
             if g.read(16):
                 return
             
-        self.f.write(struct.pack(self.endian+"IHHIIII", 0xa1b2c3d4L,
+        self.f.write(struct.pack(self.endian+"IHHIIII", 0xa1b23c4dL if self.nano else 0xa1b2c3d4L,
                                  2, 4, 0, 0, MTU, self.linktype))
         self.f.flush()
     
@@ -958,7 +1017,7 @@ def _write_packet(self, packet, sec=None, usec=None, caplen=None, wirelen=None):
             if sec is None:
                 sec = it
             if usec is None:
-                usec = int(round((t-it)*1000000))
+                usec = int(round((t - it) * (1000000000 if self.nano else 1000000)))
         self.f.write(struct.pack(self.endian+"IIII", sec, usec, caplen, wirelen))
         self.f.write(packet)
         if self.sync:
@@ -996,7 +1055,7 @@ def _write_packet(self, packet):
                 self._write_packet(pkt)
             return
         sec = int(packet.time)
-        usec = int(round((packet.time-sec)*1000000))
+        usec = int(round((packet.time - sec) * (1000000000 if self.nano else 1000000)))
         s = str(packet)
         caplen = len(s)
         RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen)


Test patch:
diff --git a/test/regression.uts b/test/regression.uts
--- a/test/regression.uts
+++ b/test/regression.uts
@@ -4609,6 +4609,7 @@ nf5.version == 5 and nf5[NetflowHeaderV5].count == 2 and isinstance(nf5[NetflowR
 import cStringIO
 pcapfile = cStringIO.StringIO('\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacVo*\n\x00(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV_-\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\xf90\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00')
 pcapngfile = cStringIO.StringIO('\n\r\r\n\\\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00,\x00File created by merging: \nFile1: test.pcap \n\x04\x00\x08\x00mergecap\x00\x00\x00\x00\\\x00\x00\x00\x01\x00\x00\x00\\\x00\x00\x00e\x00\x00\x00\xff\xff\x00\x00\x02\x006\x00Unknown/not available in original file format(libpcap)\x00\x00\t\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\\\x00\x00\x00\x06\x00\x00\x00H\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00/\xfc[\xcd(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00H\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\x1f\xff[\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r<\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\xb9\x02\\\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00<\x00\x00\x00')
+pcapnanofile = cStringIO.StringIO("M<\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacV\xc9\xc1\xb5'(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV-;\xc1'\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\x9aL\xcf'\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00")
 
 = Read a pcap file
 pktpcap = rdpcap(pcapfile)
@@ -4616,17 +4617,17 @@ pktpcap = rdpcap(pcapfile)
 = Read a pcapng file
 pktpcapng = rdpcap(pcapngfile)
 
-= Check both packet lists are the same
-assert list(pktpcap) == list(pktpcapng)
+= Read a pcap file with nanosecond precision
+pktpcapnano = rdpcap(pcapnanofile)
+
+= Check all packet lists are the same
+assert list(pktpcap) == list(pktpcapng) == list(pktpcapnano)
+assert [p.time for p in pktpcap] == [p.time for p in pktpcapng] == [p.time for p in pktpcapnano]
 
 = Check packets from pcap file
 assert all(IP in pkt for pkt in pktpcap)
 assert all(any(proto in pkt for pkt in pktpcap) for proto in [ICMP, UDP, TCP])
 
-= Check packets from pcap file
-assert all(IP in pkt for pkt in pktpcapng)
-assert all(any(proto in pkt for pkt in pktpcapng) for proto in [ICMP, UDP, TCP])
-
 = Check wrpcap()
 import os, tempfile
 fdesc, filename = tempfile.mkstemp()
@@ -4656,6 +4657,55 @@ fdesc.close()
 assert list(pktpcap[TCP]) == list(pktpcap_tcp)
 os.unlink(filename)
 
+= Check wrpcap(nano=True)
+fdesc, filename = tempfile.mkstemp()
+fdesc = os.fdopen(fdesc, "w")
+pktpcapnano[0].time += 0.000000001
+wrpcap(fdesc, pktpcapnano, nano=True)
+fdesc.close()
+pktpcapnanoread = rdpcap(filename)
+assert pktpcapnanoread[0].time == pktpcapnano[0].time
+assert pktpcapnanoread[0].time == pktpcap[0].time + 0.000000001
+os.unlink(filename)
+
+= Check PcapNg with nanosecond precision using obsolete packet block
+* first packet from capture file icmp2.ntar -- https://wiki.wireshark.org/Development/PcapNg?action=AttachFile&do=view&target=icmp2.ntar
+pcapngfile = cStringIO.StringIO('\n\r\r\n\x1c\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xa8\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x01\x00\x00\x00(\x00\x00\x00\x01\x00\x00\x00\xff\xff\x00\x00\r\x00\x01\x00\x04\x04K\x00\t\x00\x01\x00\tK=N\x00\x00\x00\x00(\x00\x00\x00\x02\x00\x00\x00n\x00\x00\x00\x00\x00\x00\x00e\x14\x00\x00)4\'ON\x00\x00\x00N\x00\x00\x00\x00\x12\xf0\x11h\xd6\x00\x13r\t{\xea\x08\x00E\x00\x00<\x90\xa1\x00\x00\x80\x01\x8e\xad\xc0\xa8M\x07\xc0\xa8M\x1a\x08\x00r[\x03\x00\xd8\x00abcdefghijklmnopqrstuvwabcdefghi\xeay$\xf6\x00\x00n\x00\x00\x00')
+pktpcapng = rdpcap(pcapngfile)
+assert len(pktpcapng) == 1
+pkt = pktpcapng[0]
+# weird, but wireshark agrees
+assert pkt.time == 22425.352221737
+assert isinstance(pkt, Ether)
+pkt = pkt.payload
+assert isinstance(pkt, IP)
+pkt = pkt.payload
+assert isinstance(pkt, ICMP)
+pkt = pkt.payload
+assert isinstance(pkt, Raw) and pkt.load == 'abcdefghijklmnopqrstuvwabcdefghi'
+pkt = pkt.payload
+assert isinstance(pkt, Padding) and pkt.load == '\xeay$\xf6'
+pkt = pkt.payload
+assert isinstance(pkt, NoPayload)
+
+= Check PcapNg using Simple Packet Block
+* previous file with the (obsolete) packet block replaced by a Simple Packet Block
+pcapngfile = cStringIO.StringIO('\n\r\r\n\x1c\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xa8\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x01\x00\x00\x00(\x00\x00\x00\x01\x00\x00\x00\xff\xff\x00\x00\r\x00\x01\x00\x04\x04K\x00\t\x00\x01\x00\tK=N\x00\x00\x00\x00(\x00\x00\x00\x03\x00\x00\x00`\x00\x00\x00N\x00\x00\x00\x00\x12\xf0\x11h\xd6\x00\x13r\t{\xea\x08\x00E\x00\x00<\x90\xa1\x00\x00\x80\x01\x8e\xad\xc0\xa8M\x07\xc0\xa8M\x1a\x08\x00r[\x03\x00\xd8\x00abcdefghijklmnopqrstuvwabcdefghi\xeay$\xf6\x00\x00`\x00\x00\x00')
+pktpcapng = rdpcap(pcapngfile)
+assert len(pktpcapng) == 1
+pkt = pktpcapng[0]
+assert isinstance(pkt, Ether)
+pkt = pkt.payload
+assert isinstance(pkt, IP)
+pkt = pkt.payload
+assert isinstance(pkt, ICMP)
+pkt = pkt.payload
+assert isinstance(pkt, Raw) and pkt.load == 'abcdefghijklmnopqrstuvwabcdefghi'
+pkt = pkt.payload
+assert isinstance(pkt, Padding) and pkt.load == '\xeay$\xf6'
+pkt = pkt.payload
+assert isinstance(pkt, NoPayload)
+
 
 ############
 ############