CVE-2019-11939: Allocation of Resources Without Limits or Throttling (CWE-770)
value: Golang Facebook Thrift servers would not error upon receiving messages declaring containers of sizes larger than the payload. As a result, malicious clients could send short messages which would result in a large memory allocation, potentially leading to denial of service. This issue affects Facebook Thrift prior to v2020.03.16.00.
CWE description: The product allocates a reusable resource or group of resources on behalf of an actor without imposing any restrictions on the size or number of resources that can be allocated, in violation of the intended security policy for that actor.
CVSS v3 base score: 7.5
Published: 2020-03-18T01:15Z
Repository: https://github.com/facebook/fbthrift
Language: Go
Files changed: thrift/lib/go/thrift/binary_protocol.go, thrift/lib/go/thrift/binary_protocol_test.go, thrift/lib/go/thrift/compact_protocol.go, thrift/lib/go/thrift/compact_protocol_test.go, thrift/lib/go/thrift/http_client.go, thrift/lib/go/thrift/protocol_test.go
Commit date: 2020-03-11 01:08:07 -0700
Commit message: Validate container size against total message size

Summary:
For performance reasons we preallocate golang containers with
the size given in the field header.  This allowed an attacker to
trigger very large memory allocations and potentially crash the server
with small messages.  Before creating the golang container confirm
that the message is theoretically large enough to contain a list/map/set
of the given size.

This requires that the binary and compact protocols use transports
that can expose the amount of data waiting to be read.  As a result of
this change you will not be able to do things like use the raw socket
transport or talk to endpoints over the HTTP transport that don't send
a content length header.

Fixes CVE-2019-11939.

Differential Revision: D19595758

fbshipit-source-id: 48bb9dbaf0467cea7a54602f0b07b00a8755c3f9

Diff: diff --git a/thrift/lib/go/thrift/binary_protocol.go b/thrift/lib/go/thrift/binary_protocol.go
index e82da9975a..2b306f4465 100644
--- a/thrift/lib/go/thrift/binary_protocol.go
+++ b/thrift/lib/go/thrift/binary_protocol.go
@@ -327,10 +327,14 @@ func (p *BinaryProtocol) ReadMapBegin() (kType, vType Type, size int, err error)
 	}
 	if size32 < 0 {
 		err = invalidDataLength
 		return
 	}
+	if uint64(size32*2) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
+		err = invalidDataLength
+		return
+	}
 	size = int(size32)
 	return kType, vType, size, nil
 }
 
 func (p *BinaryProtocol) ReadMapEnd() error {
@@ -351,10 +355,14 @@ func (p *BinaryProtocol) ReadListBegin() (elemType Type, size int, err error) {
 	}
 	if size32 < 0 {
 		err = invalidDataLength
 		return
 	}
+	if uint64(size32) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
+		err = invalidDataLength
+		return
+	}
 	size = int(size32)
 
 	return
 }
 
@@ -376,10 +384,14 @@ func (p *BinaryProtocol) ReadSetBegin() (elemType Type, size int, err error) {
 	}
 	if size32 < 0 {
 		err = invalidDataLength
 		return
 	}
+	if uint64(size32) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
+		err = invalidDataLength
+		return
+	}
 	size = int(size32)
 	return elemType, size, nil
 }
 
 func (p *BinaryProtocol) ReadSetEnd() error {
@@ -454,11 +466,11 @@ func (p *BinaryProtocol) ReadBinary() ([]byte, error) {
 		return nil, e
 	}
 	if size < 0 {
 		return nil, invalidDataLength
 	}
-	if uint64(size) > p.trans.RemainingBytes() {
+	if uint64(size) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
 		return nil, invalidDataLength
 	}
 
 	isize := int(size)
 	buf := make([]byte, isize)
@@ -485,11 +497,11 @@ func (p *BinaryProtocol) readAll(buf []byte) error {
 
 func (p *BinaryProtocol) readStringBody(size int32) (value string, err error) {
 	if size < 0 {
 		return "", nil
 	}
-	if uint64(size) > p.trans.RemainingBytes() {
+	if uint64(size) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
 		return "", invalidDataLength
 	}
 	var buf []byte
 	if int(size) <= len(p.buffer) {
 		buf = p.buffer[0:size]
diff --git a/thrift/lib/go/thrift/binary_protocol_test.go b/thrift/lib/go/thrift/binary_protocol_test.go
index 3f8854ee58..51dc797a79 100644
--- a/thrift/lib/go/thrift/binary_protocol_test.go
+++ b/thrift/lib/go/thrift/binary_protocol_test.go
@@ -15,10 +15,12 @@
  */
 
 package thrift
 
 import (
+	"bytes"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
 )
 
@@ -29,19 +31,68 @@ func TestReadWriteBinaryProtocol(t *testing.T) {
 func TestSkipUnknownTypeBinaryProtocol(t *testing.T) {
 	var m MyTestStruct
 	d := NewDeserializer()
 	f := NewBinaryProtocolFactoryDefault()
 	d.Protocol = f.GetProtocol(d.Transport)
-	// skip over a map with invalid key/value type and 1.7B entries
-	data := []byte("\n\x10\rO\t6\x03\n\n\n\x10\r\n\tslice\x00")
+	// skip over a map with invalid key/value type and ~550M entries
+	data := make([]byte, 1100000000)
+	copy(data[:], []byte("\n\x10\rO\t6\x03\n\n\n\x10\r\n\tsl ce\x00"))
+	transport, _ := d.Transport.(*MemoryBuffer)
+	transport.Buffer = bytes.NewBuffer(data)
 	start := time.Now()
-	err := d.Read(&m, data)
+	err := m.Read(d.Protocol)
 	if err == nil {
 		t.Fatalf("Parsed invalid message correctly")
 	} else if !strings.Contains(err.Error(), "unknown type") {
 		t.Fatalf("Failed for reason besides unknown type")
 	}
 
 	if time.Now().Sub(start).Seconds() > 5 {
 		t.Fatalf("It should not take seconds to parse a small message")
 	}
 }
+
+func TestInitialAllocationMapBinaryProtocol(t *testing.T) {
+	var m MyTestStruct
+	d := NewDeserializer()
+	f := NewBinaryProtocolFactoryDefault()
+	d.Protocol = f.GetProtocol(d.Transport)
+	// attempts to allocate a map with 1.8B elements for a 20 byte message
+	data := []byte("\n\x10\rO\t6\x03\n\n\n\x10\r\n\tslice\x00")
+	err := d.Read(&m, data)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
+
+func TestInitialAllocationListBinaryProtocol(t *testing.T) {
+	var m MyTestStruct
+	d := NewDeserializer()
+	f := NewBinaryProtocolFactoryDefault()
+	d.Protocol = f.GetProtocol(d.Transport)
+	// attempts to allocate a list with 1.8B elements for a 20 byte message
+	data := []byte("\n\x10\rO\t6\x03\n\n\n\x10\x0f\n\tslice\x00")
+	err := d.Read(&m, data)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
+
+func TestInitialAllocationSetBinaryProtocol(t *testing.T) {
+	var m MyTestStruct
+	d := NewDeserializer()
+	f := NewBinaryProtocolFactoryDefault()
+	d.Protocol = f.GetProtocol(d.Transport)
+	// attempts to allocate a set with 1.8B elements for a 20 byte message
+	data := []byte("\n\x12\rO\t6\x03\n\n\n\x10\x0e\n\tslice\x00")
+	err := d.Read(&m, data)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		fmt.Printf("Got %+v", err)
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
diff --git a/thrift/lib/go/thrift/compact_protocol.go b/thrift/lib/go/thrift/compact_protocol.go
index 2928b1ef7f..f01cf2eae2 100644
--- a/thrift/lib/go/thrift/compact_protocol.go
+++ b/thrift/lib/go/thrift/compact_protocol.go
@@ -410,11 +410,10 @@ func (p *CompactProtocol) ReadFieldBegin() (name string, typeId Type, id int16,
 
 	// if it's a stop, then we can return immediately, as the struct is over.
 	if (t & 0x0f) == STOP {
 		return "", STOP, 0, nil
 	}
-
 	// mask off the 4 MSB of the type header. it could contain a field id delta.
 	modifier := int16((t & 0xf0) >> 4)
 	if modifier == 0 {
 		// not a delta. look ahead for the zigzag varint field id.
 		id, err = p.ReadI16()
@@ -456,10 +455,14 @@ func (p *CompactProtocol) ReadMapBegin() (keyType Type, valueType Type, size int
 	}
 	if size32 < 0 {
 		err = invalidDataLength
 		return
 	}
+	if uint64(size32*2) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
+		err = invalidDataLength
+		return
+	}
 	size = int(size32)
 
 	keyAndValueType := byte(STOP)
 	if size != 0 {
 		keyAndValueType, err = p.readByteDirect()
@@ -494,10 +497,15 @@ func (p *CompactProtocol) ReadListBegin() (elemType Type, size int, err error) {
 			err = invalidDataLength
 			return
 		}
 		size = int(size2)
 	}
+	if uint64(size) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
+		err = invalidDataLength
+		return
+	}
+
 	elemType, e := p.getType(compactType(size_and_type))
 	if e != nil {
 		err = NewProtocolException(e)
 		return
 	}
@@ -594,11 +602,11 @@ func (p *CompactProtocol) ReadString() (value string, err error) {
 		return "", NewProtocolException(e)
 	}
 	if length < 0 {
 		return "", invalidDataLength
 	}
-	if uint64(length) > p.trans.RemainingBytes() {
+	if uint64(length) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
 		return "", invalidDataLength
 	}
 
 	if length == 0 {
 		return "", nil
@@ -623,11 +631,11 @@ func (p *CompactProtocol) ReadBinary() (value []byte, err error) {
 		return []byte{}, nil
 	}
 	if length < 0 {
 		return nil, invalidDataLength
 	}
-	if uint64(length) > p.trans.RemainingBytes() {
+	if uint64(length) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
 		return nil, invalidDataLength
 	}
 
 	buf := make([]byte, length)
 	_, e = io.ReadFull(p.trans, buf)
diff --git a/thrift/lib/go/thrift/compact_protocol_test.go b/thrift/lib/go/thrift/compact_protocol_test.go
index 1fef763d02..a523401d46 100644
--- a/thrift/lib/go/thrift/compact_protocol_test.go
+++ b/thrift/lib/go/thrift/compact_protocol_test.go
@@ -16,20 +16,20 @@
 
 package thrift
 
 import (
 	"bytes"
+	"strings"
 	"testing"
 )
 
 func TestReadWriteCompactProtocol(t *testing.T) {
 	ReadWriteProtocolTest(t, NewCompactProtocolFactory())
 	// CompactProtocol is capable of reading and writing in different goroutines.
 	ReadWriteProtocolParallelTest(t, NewCompactProtocolFactory())
 	transports := []Transport{
 		NewMemoryBuffer(),
-		NewStreamTransportRW(bytes.NewBuffer(make([]byte, 0, 16384))),
 		NewFramedTransport(NewMemoryBuffer()),
 	}
 	for _, trans := range transports {
 		p := NewCompactProtocol(trans)
 		ReadWriteBool(t, p, trans)
@@ -52,5 +52,67 @@ func TestReadWriteCompactProtocol(t *testing.T) {
 		p = NewCompactProtocol(trans)
 		ReadWriteStruct(t, p, trans)
 		trans.Close()
 	}
 }
+
+func TestInitialAllocationMapCompactProtocol(t *testing.T) {
+	var m MyTestStruct
+	d := NewDeserializer()
+	f := NewCompactProtocolFactory()
+	d.Protocol = f.GetProtocol(d.Transport)
+	// attempts to allocate a map of 930M elements for a 9 byte message
+	data := []byte("%0\x88\x8a\x97\xb7\xc4\x030")
+	err := d.Read(&m, data)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
+
+func TestInitialAllocationListCompactProtocol(t *testing.T) {
+	var m MyTestStruct
+	d := NewDeserializer()
+	f := NewCompactProtocolFactory()
+	d.Protocol = f.GetProtocol(d.Transport)
+	// attempts to allocate a list of 950M elements for an 11 byte message
+	data := []byte("%0\x98\xfa\xb7\xb7\xc4\xc4\x03\x01a")
+	err := d.Read(&m, data)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
+
+func TestInitialAllocationSetCompactProtocol(t *testing.T) {
+	var m MyTestStruct
+	d := NewDeserializer()
+	f := NewCompactProtocolFactory()
+	d.Protocol = f.GetProtocol(d.Transport)
+	// attempts to allocate a list of 950M elements for an 11 byte message
+	data := []byte("%0\xa8\xfa\x97\xb7\xc4\xc4\x03\x01a")
+	err := d.Read(&m, data)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
+
+func TestInitialAllocationMapCompactProtocolLimitedR(t *testing.T) {
+	var m MyTestStruct
+
+	// attempts to allocate a map of 930M elements for a 9 byte message
+	data := []byte("%0\x88\x8a\x97\xb7\xc4\x030")
+	p := NewCompactProtocol(
+		NewStreamTransportLimitedR(bytes.NewBuffer(data), len(data)),
+	)
+
+	err := m.Read(p)
+	if err == nil {
+		t.Fatalf("Parsed invalid message correctly")
+	} else if !strings.Contains(err.Error(), "Invalid data length") {
+		t.Fatalf("Failed for reason besides Invalid data length")
+	}
+}
diff --git a/thrift/lib/go/thrift/http_client.go b/thrift/lib/go/thrift/http_client.go
index c2d5104f11..41a33f598c 100644
--- a/thrift/lib/go/thrift/http_client.go
+++ b/thrift/lib/go/thrift/http_client.go
@@ -17,11 +17,10 @@
 package thrift
 
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
 )
 
@@ -33,10 +32,11 @@ var DefaultHTTPClient *http.Client = http.DefaultClient
 type HTTPClient struct {
 	client             *http.Client
 	response           *http.Response
 	url                *url.URL
 	requestBuffer      *bytes.Buffer
+	responseBuffer     bytes.Buffer
 	header             http.Header
 	nsecConnectTimeout int64
 	nsecReadTimeout    int64
 }
 
@@ -162,24 +162,13 @@ func (p *HTTPClient) Open() error {
 func (p *HTTPClient) IsOpen() bool {
 	return p.response != nil || p.requestBuffer != nil
 }
 
 func (p *HTTPClient) closeResponse() error {
-	var err error
-	if p.response != nil && p.response.Body != nil {
-		// The docs specify that if keepalive is enabled and the response body is not
-		// read to completion the connection will never be returned to the pool and
-		// reused. Errors are being ignored here because if the connection is invalid
-		// and this fails for some reason, the Close() method will do any remaining
-		// cleanup.
-		io.Copy(ioutil.Discard, p.response.Body)
-
-		err = p.response.Body.Close()
-	}
-
 	p.response = nil
-	return err
+	p.responseBuffer.Reset()
+	return nil
 }
 
 func (p *HTTPClient) Close() error {
 	if p.requestBuffer != nil {
 		p.requestBuffer.Reset()
@@ -190,19 +179,19 @@ func (p *HTTPClient) Close() error {
 
 func (p *HTTPClient) Read(buf []byte) (int, error) {
 	if p.response == nil {
 		return 0, NewTransportException(NOT_OPEN, "Response buffer is empty, no request.")
 	}
-	n, err := p.response.Body.Read(buf)
+	n, err := p.responseBuffer.Read(buf)
 	if n > 0 && (err == nil || err == io.EOF) {
 		return n, nil
 	}
 	return n, NewTransportExceptionFromError(err)
 }
 
 func (p *HTTPClient) ReadByte() (c byte, err error) {
-	return readByte(p.response.Body)
+	return readByte(&p.responseBuffer)
 }
 
 func (p *HTTPClient) Write(buf []byte) (int, error) {
 	n, err := p.requestBuffer.Write(buf)
 	return n, err
@@ -228,26 +217,30 @@ func (p *HTTPClient) Flush() error {
 	req.Header = p.header
 	response, err := p.client.Do(req)
 	if err != nil {
 		return NewTransportExceptionFromError(err)
 	}
+
+	defer response.Body.Close()
+
 	if response.StatusCode != http.StatusOK {
 		// Close the response to avoid leaking file descriptors. closeResponse does
 		// more than just call Close(), so temporarily assign it and reuse the logic.
 		p.response = response
 		p.closeResponse()
 
 		// TODO(pomack) log bad response
 		return NewTransportException(UNKNOWN_TRANSPORT_EXCEPTION, "HTTP Response code: "+strconv.Itoa(response.StatusCode))
 	}
+
+	_, err = io.Copy(&p.responseBuffer, response.Body)
+	if err != nil {
+		return NewTransportExceptionFromError(err)
+	}
+
 	p.response = response
 	return nil
 }
 
 func (p *HTTPClient) RemainingBytes() (num_bytes uint64) {
-	len := p.response.ContentLength
-	if len >= 0 {
-		return uint64(len)
-	}
-
-	return UnknownRemaining // the truth is, we just don't know unless framed is used
+	return uint64(p.responseBuffer.Len())
 }
diff --git a/thrift/lib/go/thrift/protocol_test.go b/thrift/lib/go/thrift/protocol_test.go
index cdbb8acbd3..94d75a3ff8 100644
--- a/thrift/lib/go/thrift/protocol_test.go
+++ b/thrift/lib/go/thrift/protocol_test.go
@@ -15,11 +15,10 @@
  */
 
 package thrift
 
 import (
-	"bytes"
 	"io"
 	"io/ioutil"
 	"math"
 	"net"
 	"net/http"
@@ -170,12 +169,10 @@ type protocolTest func(t testing.TB, p Protocol, trans Transport)
 // implementations require that the data exists to be read when they are called (like bytes.Buffer)
 func ReadWriteProtocolParallelTest(t *testing.T, protocolFactory ProtocolFactory) {
 	rConn, wConn := tcpStreamSetupForTest(t)
 	rdr, writer := io.Pipe()
 	transports := []TransportFactory{
-		NewStreamTransportFactory(rdr, writer, false),                             // use a pipe
-		NewStreamTransportFactory(rConn, wConn, false),                            // use tcp over network
 		NewFramedTransportFactory(NewStreamTransportFactory(rdr, writer, false)),  // framed over pipe
 		NewFramedTransportFactory(NewStreamTransportFactory(rConn, wConn, false)), // framed over tcp
 	}
 	const iterations = 100
 
@@ -235,16 +232,14 @@ func ReadWriteProtocolParallelTest(t *testing.T, protocolFactory ProtocolFactory
 		WriteStruct(t, p, trans)
 	})
 }
 
 func ReadWriteProtocolTest(t *testing.T, protocolFactory ProtocolFactory) {
-	buf := bytes.NewBuffer(make([]byte, 0, 1024))
 	l := HTTPClientSetupForTest(t)
 	defer l.Close()
 	transports := []TransportFactory{
 		NewMemoryBufferTransportFactory(1024),
-		NewStreamTransportFactory(buf, buf, true),
 		NewFramedTransportFactory(NewMemoryBufferTransportFactory(1024)),
 		NewHTTPPostClientTransportFactory("http://" + l.Addr().String()),
 	}
 
 	doForAllTransports := func(protTest protocolTest) {

Vulnerable code: 	if uint64(size) > p.trans.RemainingBytes() {
	if uint64(size) > p.trans.RemainingBytes() {
	// skip over a map with invalid key/value type and 1.7B entries
	data := []byte("\n\x10\rO\t6\x03\n\n\n\x10\r\n\tslice\x00")
	err := d.Read(&m, data)

	if uint64(length) > p.trans.RemainingBytes() {
	if uint64(length) > p.trans.RemainingBytes() {
		NewStreamTransportRW(bytes.NewBuffer(make([]byte, 0, 16384))),
	"io/ioutil"
	var err error
	if p.response != nil && p.response.Body != nil {
		// The docs specify that if keepalive is enabled and the response body is not
		// read to completion the connection will never be returned to the pool and
		// reused. Errors are being ignored here because if the connection is invalid
		// and this fails for some reason, the Close() method will do any remaining
		// cleanup.
		io.Copy(ioutil.Discard, p.response.Body)

		err = p.response.Body.Close()
	}

	return err
	n, err := p.response.Body.Read(buf)
	return readByte(p.response.Body)
	len := p.response.ContentLength
	if len >= 0 {
		return uint64(len)
	}

	return UnknownRemaining // the truth is, we just don't know unless framed is used
	"bytes"
		NewStreamTransportFactory(rdr, writer, false),                             // use a pipe
		NewStreamTransportFactory(rConn, wConn, false),                            // use tcp over network
	buf := bytes.NewBuffer(make([]byte, 0, 1024))
		NewStreamTransportFactory(buf, buf, true),
Fixed code: 	if uint64(size32*2) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
		err = invalidDataLength
		return
	}
	if uint64(size32) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
		err = invalidDataLength
		return
	}
	if uint64(size32) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
		err = invalidDataLength
		return
	}
	if uint64(size) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
	if uint64(size) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
	"bytes"
	"fmt"
	// skip over a map with invalid key/value type and ~550M entries
	data := make([]byte, 1100000000)
	copy(data[:], []byte("\n\x10\rO\t6\x03\n\n\n\x10\r\n\tsl ce\x00"))
	transport, _ := d.Transport.(*MemoryBuffer)
	transport.Buffer = bytes.NewBuffer(data)
	err := m.Read(d.Protocol)

func TestInitialAllocationMapBinaryProtocol(t *testing.T) {
	var m MyTestStruct
	d := NewDeserializer()
	f := NewBinaryProtocolFactoryDefault()
	d.Protocol = f.GetProtocol(d.Transport)
	// attempts to allocate a map with 1.8B elements for a 20 byte message
	data := []byte("\n\x10\rO\t6\x03\n\n\n\x10\r\n\tslice\x00")
	err := d.Read(&m, data)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}

func TestInitialAllocationListBinaryProtocol(t *testing.T) {
	var m MyTestStruct
	d := NewDeserializer()
	f := NewBinaryProtocolFactoryDefault()
	d.Protocol = f.GetProtocol(d.Transport)
	// attempts to allocate a list with 1.8B elements for a 20 byte message
	data := []byte("\n\x10\rO\t6\x03\n\n\n\x10\x0f\n\tslice\x00")
	err := d.Read(&m, data)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}

func TestInitialAllocationSetBinaryProtocol(t *testing.T) {
	var m MyTestStruct
	d := NewDeserializer()
	f := NewBinaryProtocolFactoryDefault()
	d.Protocol = f.GetProtocol(d.Transport)
	// attempts to allocate a set with 1.8B elements for a 20 byte message
	data := []byte("\n\x12\rO\t6\x03\n\n\n\x10\x0e\n\tslice\x00")
	err := d.Read(&m, data)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		fmt.Printf("Got %+v", err)
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}
	if uint64(size32*2) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
		err = invalidDataLength
		return
	}
	if uint64(size) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
		err = invalidDataLength
		return
	}

	if uint64(length) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
	if uint64(length) > p.trans.RemainingBytes() || p.trans.RemainingBytes() == UnknownRemaining {
	"strings"

func TestInitialAllocationMapCompactProtocol(t *testing.T) {
	var m MyTestStruct
	d := NewDeserializer()
	f := NewCompactProtocolFactory()
	d.Protocol = f.GetProtocol(d.Transport)
	// attempts to allocate a map of 930M elements for a 9 byte message
	data := []byte("%0\x88\x8a\x97\xb7\xc4\x030")
	err := d.Read(&m, data)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}

func TestInitialAllocationListCompactProtocol(t *testing.T) {
	var m MyTestStruct
	d := NewDeserializer()
	f := NewCompactProtocolFactory()
	d.Protocol = f.GetProtocol(d.Transport)
	// attempts to allocate a list of 950M elements for an 11 byte message
	data := []byte("%0\x98\xfa\xb7\xb7\xc4\xc4\x03\x01a")
	err := d.Read(&m, data)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}

func TestInitialAllocationSetCompactProtocol(t *testing.T) {
	var m MyTestStruct
	d := NewDeserializer()
	f := NewCompactProtocolFactory()
	d.Protocol = f.GetProtocol(d.Transport)
	// attempts to allocate a list of 950M elements for an 11 byte message
	data := []byte("%0\xa8\xfa\x97\xb7\xc4\xc4\x03\x01a")
	err := d.Read(&m, data)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}

func TestInitialAllocationMapCompactProtocolLimitedR(t *testing.T) {
	var m MyTestStruct

	// attempts to allocate a map of 930M elements for a 9 byte message
	data := []byte("%0\x88\x8a\x97\xb7\xc4\x030")
	p := NewCompactProtocol(
		NewStreamTransportLimitedR(bytes.NewBuffer(data), len(data)),
	)

	err := m.Read(p)
	if err == nil {
		t.Fatalf("Parsed invalid message correctly")
	} else if !strings.Contains(err.Error(), "Invalid data length") {
		t.Fatalf("Failed for reason besides Invalid data length")
	}
}
	responseBuffer     bytes.Buffer
	p.responseBuffer.Reset()
	return nil
	n, err := p.responseBuffer.Read(buf)
	return readByte(&p.responseBuffer)

	defer response.Body.Close()


	_, err = io.Copy(&p.responseBuffer, response.Body)
	if err != nil {
		return NewTransportExceptionFromError(err)
	}

	return uint64(p.responseBuffer.Len())