Repository: secdev/scapy

Issue:
on-the-wire packet length lost

[scapyd.pcap.gz](https://github.com/secdev/scapy/files/1656955/scapyd.pcap.gz)

[orig.pcap.gz](https://github.com/secdev/scapy/files/1656954/orig.pcap.gz)
If one reads-in a pcap and then writes it out again, the on-the-wire packet length field of the per-packet pcap header gets set to the snaplen.  This causes issues with other utilities which sanity check packet headers against the on-the-wire length.

For example, reading the original capture in tcpdump:
$ tcpdump -n -r orig.pcap 
reading from file orig.pcap, link-type EN10MB (Ethernet)
11:26:29.212874 IP 10.138.0.5.54902 > 10.138.0.6.65432: Flags [S], seq 2786883070, win 28400, options [mss 1420,sackOK,TS val 2037739231 ecr 0,nop,wscale 7], length 0
11:26:29.212936 IP 10.138.0.6.65432 > 10.138.0.5.54902: Flags [S.], seq 358205320, ack 2786883071, win 28160, options [mss 1420,sackOK,TS val 480578934 ecr 2037739231,nop,wscale 7], length 0
11:26:29.212944 IP 10.138.0.5.54902 > 10.138.0.6.65432: Flags [.], ack 1, win 222, options [nop,nop,TS val 2037739231 ecr 480578934], length 0
11:26:29.212958 IP 10.138.0.5.54902 > 10.138.0.6.65432: Flags [.], seq 1:7041, ack 1, win 222, options [nop,nop,TS val 2037739231 ecr 480578934], length 7040
11:26:29.212962 IP 10.138.0.5.54902 > 10.138.0.6.65432: Flags [.], seq 7041:14081, ack 1, win 222, options [nop,nop,TS val 2037739231 ecr 480578934], length 7040


And then after that pcap was read-in by scapy and written-out to another file:

$ tcpdump -n -r scapyd.pcap 
reading from file scapyd.pcap, link-type EN10MB (Ethernet)
11:26:29.212874 IP 10.138.0.5.54902 > 10.138.0.6.65432: Flags [S], seq 2786883070, win 28400, options [mss 1420,sackOK,TS val 2037739231 ecr 0,nop,wscale 7], length 0
11:26:29.212936 IP 10.138.0.6.65432 > 10.138.0.5.54902: Flags [S.], seq 358205320, ack 2786883071, win 28160, options [mss 1420,sackOK,TS val 480578934 ecr 2037739231,nop,wscale 7], length 0
11:26:29.212944 IP 10.138.0.5.54902 > 10.138.0.6.65432: Flags [.], ack 1, win 222, options [nop,nop,TS val 2037739231 ecr 480578934], length 0
11:26:29.212958 IP truncated-ip - 7010 bytes missing! 10.138.0.5.54902 > 10.138.0.6.65432: Flags [.], seq 1:7041, ack 1, win 222, options [nop,nop,TS val 2037739231 ecr 480578934], length 7040
11:26:29.212962 IP truncated-ip - 7010 bytes missing! 10.138.0.5.54902 > 10.138.0.6.65432: Flags [.], seq 7041:14081, ack 1, win 222, options [nop,nop,TS val 2037739231 ecr 480578934], length 7040

This also affects the willingness of utilities such as tshark to compute RTTs because the on-the-wire length stored in the per-packet pcap header does not match the packet lengths in the packet headers themselves.




Discussion:

Resolving patch:
diff --git a/scapy/packet.py b/scapy/packet.py
--- a/scapy/packet.py
+++ b/scapy/packet.py
@@ -56,7 +56,9 @@ class Packet(six.with_metaclass(Packet_metaclass, BasePacket)):
         # used for sr()
         "_answered",
         # used when sniffing
-        "direction", "sniffed_on"
+        "direction", "sniffed_on",
+        # handle snaplen Vs real length
+        "wirelen",
     ]
     name = None
     fields_desc = []
@@ -123,6 +125,7 @@ def __init__(self, _pkt=b"", post_transform=None, _internal=0, _underlayer=None,
         self.explicit = 0
         self.raw_packet_cache = None
         self.raw_packet_cache_fields = None
+        self.wirelen = None
         if _pkt:
             self.dissect(_pkt)
             if not _internal:
@@ -202,6 +205,7 @@ def copy(self):
         clone.raw_packet_cache_fields = self.copy_fields_dict(
             self.raw_packet_cache_fields
         )
+        clone.wirelen = self.wirelen
         clone.post_transforms = self.post_transforms[:]
         clone.payload = self.payload.copy()
         clone.payload.add_underlayer(clone)
@@ -245,6 +249,7 @@ def setfieldval(self, attr, val):
             self.explicit = 0
             self.raw_packet_cache = None
             self.raw_packet_cache_fields = None
+            self.wirelen = None
         elif attr == "payload":
             self.remove_payload()
             self.add_payload(val)
@@ -266,6 +271,7 @@ def delfieldval(self, attr):
             self.explicit = 0 # in case a default value must be explicited
             self.raw_packet_cache = None
             self.raw_packet_cache_fields = None
+            self.wirelen = None
         elif attr in self.default_fields:
             pass
         elif attr == "payload":
@@ -384,6 +390,7 @@ def self_build(self, field_pos_list=None):
                 if self.getfieldval(fname) != fval:
                     self.raw_packet_cache = None
                     self.raw_packet_cache_fields = None
+                    self.wirelen = None
                     break
             if self.raw_packet_cache is not None:
                 return self.raw_packet_cache
@@ -792,6 +799,7 @@ def clone_with(self, payload=None, **kargs):
         pkt.raw_packet_cache_fields = self.copy_fields_dict(
             self.raw_packet_cache_fields
         )
+        pkt.wirelen = self.wirelen
         if payload is not None:
             pkt.add_payload(payload)
         return pkt
diff --git a/scapy/utils.py b/scapy/utils.py
--- a/scapy/utils.py
+++ b/scapy/utils.py
@@ -826,7 +826,7 @@ def read_packet(self, size=MTU):
             return None
         sec,usec,caplen,wirelen = struct.unpack(self.endian+"IIII", hdr)
         s = self.f.read(caplen)[:size]
-        return s,(sec,usec,wirelen) # caplen = len(s)
+        return s, (sec, usec, caplen, wirelen)
 
 
     def dispatch(self, callback):
@@ -878,10 +878,10 @@ def __init__(self, filename, fdesc, magic):
             warning("PcapReader: unknown LL type [%i]/[%#x]. Using Raw packets" % (self.linktype,self.linktype))
             self.LLcls = conf.raw_layer
     def read_packet(self, size=MTU):
-        rp = RawPcapReader.read_packet(self, size=size)
+        rp = super(PcapReader, self).read_packet(size=size)
         if rp is None:
             return None
-        s,(sec,usec,wirelen) = rp
+        s, (sec, usec, caplen, wirelen) = rp
         
         try:
             p = self.LLcls(s)
@@ -892,6 +892,7 @@ def read_packet(self, size=MTU):
                 raise
             p = conf.raw_layer(s)
         p.time = sec + (0.000000001 if self.nano else 0.000001) * usec
+        p.wirelen = wirelen
         return p
     def read_all(self,count=-1):
         res = RawPcapReader.read_all(self, count)
@@ -1028,7 +1029,7 @@ def __init__(self, filename, fdesc, magic):
         RawPcapNgReader.__init__(self, filename, fdesc, magic)
 
     def read_packet(self, size=MTU):
-        rp = RawPcapNgReader.read_packet(self, size=size)
+        rp = super(PcapNgReader, self).read_packet(size=size)
         if rp is None:
             return None
         s, (linktype, tsresol, tshigh, tslow, wirelen) = rp
@@ -1042,6 +1043,7 @@ def read_packet(self, size=MTU):
             p = conf.raw_layer(s)
         if tshigh is not None:
             p.time = float((tshigh << 32) + tslow) / tsresol
+        p.wirelen = wirelen
         return p
     def read_all(self,count=-1):
         res = RawPcapNgReader.read_all(self, count)
@@ -1113,7 +1115,7 @@ def write(self, pkt):
         written to the dumpfile
 
         """
-        if isinstance(pkt, str):
+        if isinstance(pkt, bytes):
             if not self.header_present:
                 self._write_header(pkt)
             self._write_packet(pkt)
@@ -1123,7 +1125,6 @@ def write(self, pkt):
                 try:
                     p = next(pkt)
                 except StopIteration:
-                    self._write_header(b"")
                     return
                 self._write_header(p)
                 self._write_packet(p)
@@ -1187,9 +1188,10 @@ def _write_packet(self, packet):
             return
         sec = int(packet.time)
         usec = int(round((packet.time - sec) * (1000000000 if self.nano else 1000000)))
-        s = raw(packet)
-        caplen = len(s)
-        RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen)
+        rawpkt = raw(packet)
+        caplen = len(rawpkt)
+        RawPcapWriter._write_packet(self, rawpkt, sec=sec, usec=usec, caplen=caplen,
+                                    wirelen=packet.wirelen or caplen)
 
 
 re_extract_hexcap = re.compile("^((0x)?[0-9a-fA-F]{2,}[ :\t]{,3}|) *(([0-9a-fA-F]{2} {,2}){,16})")


Test patch:
diff --git a/test/regression.uts b/test/regression.uts
--- a/test/regression.uts
+++ b/test/regression.uts
@@ -5501,6 +5501,7 @@ from io import BytesIO
 pcapfile = BytesIO(b'\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacVo*\n\x00(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV_-\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\xf90\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00')
 pcapngfile = BytesIO(b'\n\r\r\n\\\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00,\x00File created by merging: \nFile1: test.pcap \n\x04\x00\x08\x00mergecap\x00\x00\x00\x00\\\x00\x00\x00\x01\x00\x00\x00\\\x00\x00\x00e\x00\x00\x00\xff\xff\x00\x00\x02\x006\x00Unknown/not available in original file format(libpcap)\x00\x00\t\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\\\x00\x00\x00\x06\x00\x00\x00H\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00/\xfc[\xcd(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00H\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\x1f\xff[\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r<\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\xb9\x02\\\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00<\x00\x00\x00')
 pcapnanofile = BytesIO(b"M<\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacV\xc9\xc1\xb5'(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV-;\xc1'\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\x9aL\xcf'\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00")
+pcapwirelenfile = BytesIO(b'\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00\x01\x00\x00\x00}\x87pZ.\xa2\x08\x00\x0f\x00\x00\x00\x10\x00\x00\x00\xff\xff\xff\xff\xff\xff GG\xee\xdd\xa8\x90\x00a')
 
 = Read a pcap file
 pktpcap = rdpcap(pcapfile)
@@ -5511,6 +5512,9 @@ pktpcapng = rdpcap(pcapngfile)
 = Read a pcap file with nanosecond precision
 pktpcapnano = rdpcap(pcapnanofile)
 
+= Read a pcap file with wirelen != captured len
+pktpcapwirelen = rdpcap(pcapwirelenfile)
+
 = Check all packet lists are the same
 assert list(pktpcap) == list(pktpcapng) == list(pktpcapnano)
 assert [p.time for p in pktpcap] == [p.time for p in pktpcapng] == [p.time for p in pktpcapnano]
@@ -5519,10 +5523,26 @@ assert [p.time for p in pktpcap] == [p.time for p in pktpcapng] == [p.time for p
 assert all(IP in pkt for pkt in pktpcap)
 assert all(any(proto in pkt for pkt in pktpcap) for proto in [ICMP, UDP, TCP])
 
-= Check wrpcap()
+= Check wirelen value from pcap file
+assert len(pktpcapwirelen) == 1
+assert pktpcapwirelen[0].wirelen is not None
+assert len(pktpcapwirelen[0]) < pktpcapwirelen[0].wirelen
+
+= Check wrpcap() then rdpcap() with wirelen
 import os, tempfile
 fdesc, filename = tempfile.mkstemp()
 fdesc = os.fdopen(fdesc, "wb")
+wrpcap(fdesc, pktpcapwirelen)
+fdesc.close()
+newpktpcapwirelen = rdpcap(filename)
+assert len(newpktpcapwirelen) == 1
+assert newpktpcapwirelen[0].wirelen is not None
+assert len(newpktpcapwirelen[0]) < newpktpcapwirelen[0].wirelen
+assert newpktpcapwirelen[0].wirelen == pktpcapwirelen[0].wirelen
+
+= Check wrpcap()
+fdesc, filename = tempfile.mkstemp()
+fdesc = os.fdopen(fdesc, "wb")
 wrpcap(fdesc, pktpcap)
 fdesc.close()