Repository: secdev/scapy

Issue:
Fix HTTP 1.1 parsing with TCP PSH flag
<!-- This is just a checklist to guide you. You can remove it safely. -->

**Checklist:**

-   [X ] If you are new to Scapy: I have checked [CONTRIBUTING.md](https://github.com/secdev/scapy/blob/master/CONTRIBUTING.md) (esp. section submitting-pull-requests)
-   [ X] I squashed commits belonging together
-   [ X] I added unit tests or explained why they are not relevant
-   [ X] I executed the regression tests for Python2 and Python3 (using `tox` or, `cd test && ./run_tests_py2, cd test && ./run_tests_py3`)
-   [ ] If the PR is still not finished, please create a [Draft Pull Request](https://github.blog/2019-02-14-introducing-draft-pull-requests/)

This PR fixes broken reassembly of HTTP messages in case a TCP PSH flag exists. Currently when reassembling HTTP messages - if content-length is not yet supplied and a TCP PSH message appears, the HTTP message will be considered complete, even though it may not be the case. An example of such communication exists in the pcap I added in test\pcaps\http_tcp_psh.pcap



Discussion:

Resolving patch:
diff --git a/scapy/layers/http.py b/scapy/layers/http.py
--- a/scapy/layers/http.py
+++ b/scapy/layers/http.py
@@ -563,8 +563,22 @@ def tcp_reassemble(cls, data, metadata):
                 # It's not Content-Length based. It could be chunked
                 encodings = http_packet[HTTP].payload._get_encodings()
                 chunked = ("chunked" in encodings)
+                is_response = isinstance(http_packet.payload, HTTPResponse)
                 if chunked:
                     detect_end = lambda dat: dat.endswith(b"\r\n\r\n")
+                # HTTP Requests that do not have any content,
+                # end with a double CRLF
+                elif isinstance(http_packet.payload, HTTPRequest):
+                    detect_end = lambda dat: dat.endswith(b"\r\n\r\n")
+                    # In case we are handling a HTTP Request,
+                    # we want to continue assessing the data,
+                    # to handle requests with a body (POST)
+                    metadata["detect_unknown"] = True
+                elif is_response and http_packet.Status_Code == b"101":
+                    # If it's an upgrade response, it may also hold a
+                    # different protocol data.
+                    # make sure all headers are present
+                    detect_end = lambda dat: dat.find(b"\r\n\r\n")
                 else:
                     # If neither Content-Length nor chunked is specified,
                     # it means it's the TCP packet that contains the data,
diff --git a/scapy/sessions.py b/scapy/sessions.py
--- a/scapy/sessions.py
+++ b/scapy/sessions.py
@@ -213,7 +213,7 @@ def _process_packet(self, pkt):
         to follow the TCP streams, and orders the fragments.
         """
         from scapy.layers.inet import IP, TCP
-        if TCP not in pkt:
+        if not pkt or TCP not in pkt:
             return pkt
         pay = pkt[TCP].payload
         if isinstance(pay, (NoPayload, conf.padding_layer)):
@@ -241,8 +241,13 @@ def _process_packet(self, pkt):
         # Note that this take care of retransmission packets.
         data.append(new_data, seq)
         # Check TCP FIN or TCP RESET
-        if pkt[TCP].flags.F or pkt[TCP].flags.R or pkt[TCP].flags.P:
+        if pkt[TCP].flags.F or pkt[TCP].flags.R:
             metadata["tcp_end"] = True
+
+        # In case any app layer protocol requires it,
+        # allow the parser to inspect TCP PSH flag
+        if pkt[TCP].flags.P:
+            metadata["tcp_psh"] = True
         # XXX TODO: check that no empty space is missing in the buffer.
         # XXX Currently, if a TCP fragment was missing, we won't notice it.
         packet = None


Test patch:
diff --git a/test/pcaps/http_tcp_psh.pcap.gz b/test/pcaps/http_tcp_psh.pcap.gz
new file mode 100644
Binary files /dev/null and b/test/pcaps/http_tcp_psh.pcap.gz differ
diff --git a/test/regression.uts b/test/regression.uts
--- a/test/regression.uts
+++ b/test/regression.uts
@@ -7595,6 +7595,21 @@ pkts[0].show()
 assert HTTPResponse in pkts[0]
 assert pkts[0].load == b'This is a test file for testing brotli decompression in Wireshark\n'
 
+= HTTP PSH bug fix
+
+tmp = "/test/pcaps/http_tcp_psh.pcap.gz"
+filename = os.path.abspath(os.path.join(os.path.dirname(__file__),"../")) + tmp
+filename = os.getenv("SCAPY_ROOT_DIR")+tmp if not os.path.exists(filename) else filename
+
+pkts = sniff(offline=filename, session=TCPSession)
+
+assert len(pkts) == 15
+# Verify a split header exists in the packet
+assert pkts[5].User_Agent == b'example_user_agent'
+
+# Verify all of the resonse data exists in the packet
+assert int(pkts[7][HTTP].Content_Length.decode()) == len(pkts[7][Raw].load)
+
 = HTTP build
 
 pkt = TCP()/HTTP()/HTTPRequest(Method=b'GET', Path=b'/download', Http_Version=b'HTTP/1.1', Accept=b'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', Accept_Encoding=b'gzip, deflate', Accept_Language=b'en-US,en;q=0.5', Cache_Control=b'max-age=0', Connection=b'keep-alive', Host=b'scapy.net', User_Agent=b'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0')