Repository: IBM/sarama (2025-08-11T16:24:33Z)

Issue:
When using lz4 for message compression and sending, it will cause memory strain on the Kafka Broker
##### Description

<!-- A clear and concise description of what the bug is and how to reproduce it. -->

##### Versions

<!-- Please specify real version numbers or git SHAs, not just "Latest" since that changes fairly regularly. -->

| Sarama | Kafka | Go  |
| ------ | ----- | --- |
|    1.45.2    |    2.8.1   |  1.24  |

##### Configuration

<!-- What configuration values are you using for Sarama and Kafka? -->

```go
	// Enable LZ4 compression with custom settings
	config.Producer.Compression = sarama.CompressionLZ4
```

##### Logs

<!-- If applicable, add logs from Sarama and/or Kafka to help explain your problem.
You can set `sarama.Logger` to a `log.Logger` to capture Sarama debug output. -->

<details><summary>logs: CLICK ME</summary>
<p>

```

```

</p>
</details>

##### Additional Context
Sarama use github.com/pierrec/lz4/v4，in this lz4 lib default block size is 4MB.It means every produce request, broker will allocate 4MB buffer to decompress request, it will cause  broker occurr a large number of gc, and request handle thread always hung at allocate buffer memory. In Issue KAFKA-10433, Broker caching the lz4 decompress buffer memory in thread local.But it also has  two problems:
1.  Old version broker when use sarama lz4 produce also has this question
2.  If broker has many request handle threads，even if use thread local buffer also cause buffer memory nervous.

<!-- Add any other context about the problem here. -->
```go
	lz4WriterPool = sync.Pool{
		New: func() interface{} {
			return lz4.NewWriter(nil)
		},
	}

func NewWriter(w io.Writer) *Writer {
	zw := &Writer{frame: lz4stream.NewFrame()}
	zw.state.init(writerStates)
	_ = zw.Apply(DefaultBlockSizeOption, DefaultChecksumOption, DefaultConcurrency, defaultOnBlockDone)
	zw.Reset(w)
	return zw
}

var (
	DefaultBlockSizeOption = BlockSizeOption(Block4Mb)
	DefaultChecksumOption  = ChecksumOption(true)
	DefaultConcurrency     = ConcurrencyOption(1)
	defaultOnBlockDone     = OnBlockDoneOption(nil)
)
```

<img width="2834" height="926" alt="Image" src="https://github.com/user-attachments/assets/3e1d67dc-b6eb-459e-acec-5a625a68d189" />


Discussion:
@lushilin hmm good catch – do you think we should be hardcoding `lz4.BlockSizeOption(lz4.Block64Kb)` to match the hardcoded block-size in the Apache Kafka Java client in [Lz4BlockOutputStream.java](https://github.com/apache/kafka/blob/4.0/clients/src/main/java/org/apache/kafka/common/compress/Lz4BlockOutputStream.java)?
@dnwe yes, i think hardcoding `lz4.BlockSizeOption(lz4.Block64Kb)` is helpful, i submit a mr for this issue. pls take a look.
#3253 https://github.com/IBM/sarama/pull/3253



Patch:
diff --git a/compress.go b/compress.go
index a7bd525bc..b752cb8a0 100644
--- a/compress.go
+++ b/compress.go
@@ -13,7 +13,11 @@ import (
 var (
 	lz4WriterPool = sync.Pool{
 		New: func() interface{} {
-			return lz4.NewWriter(nil)
+			lz := lz4.NewWriter(nil)
+			if err := lz.Apply(lz4.BlockSizeOption(lz4.Block64Kb)); err != nil {
+				panic(err)
+			}
+			return lz
 		},
 	}
 
@@ -110,85 +114,78 @@ func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) {
 	case CompressionNone:
 		return data, nil
 	case CompressionGZIP:
-		var (
-			err    error
-			buf    bytes.Buffer
-			writer *gzip.Writer
-		)
-
-		switch level {
-		case CompressionLevelDefault:
-			writer = gzipWriterPool.Get().(*gzip.Writer)
-			defer gzipWriterPool.Put(writer)
-			writer.Reset(&buf)
-		case 1:
-			writer = gzipWriterPoolForCompressionLevel1.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel1.Put(writer)
-			writer.Reset(&buf)
-		case 2:
-			writer = gzipWriterPoolForCompressionLevel2.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel2.Put(writer)
-			writer.Reset(&buf)
-		case 3:
-			writer = gzipWriterPoolForCompressionLevel3.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel3.Put(writer)
-			writer.Reset(&buf)
-		case 4:
-			writer = gzipWriterPoolForCompressionLevel4.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel4.Put(writer)
-			writer.Reset(&buf)
-		case 5:
-			writer = gzipWriterPoolForCompressionLevel5.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel5.Put(writer)
-			writer.Reset(&buf)
-		case 6:
-			writer = gzipWriterPoolForCompressionLevel6.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel6.Put(writer)
-			writer.Reset(&buf)
-		case 7:
-			writer = gzipWriterPoolForCompressionLevel7.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel7.Put(writer)
-			writer.Reset(&buf)
-		case 8:
-			writer = gzipWriterPoolForCompressionLevel8.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel8.Put(writer)
-			writer.Reset(&buf)
-		case 9:
-			writer = gzipWriterPoolForCompressionLevel9.Get().(*gzip.Writer)
-			defer gzipWriterPoolForCompressionLevel9.Put(writer)
-			writer.Reset(&buf)
-		default:
-			writer, err = gzip.NewWriterLevel(&buf, level)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if _, err := writer.Write(data); err != nil {
-			return nil, err
-		}
-		if err := writer.Close(); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
+		return gzipCompress(level, data)
 	case CompressionSnappy:
 		return snappy.Encode(data), nil
 	case CompressionLZ4:
-		writer := lz4WriterPool.Get().(*lz4.Writer)
-		defer lz4WriterPool.Put(writer)
-
-		var buf bytes.Buffer
-		writer.Reset(&buf)
-
-		if _, err := writer.Write(data); err != nil {
-			return nil, err
-		}
-		if err := writer.Close(); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
+		return lz4Compress(data)
 	case CompressionZSTD:
 		return zstdCompress(ZstdEncoderParams{level}, nil, data)
 	default:
 		return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)}
 	}
 }
+
+func gzipCompress(level int, data []byte) ([]byte, error) {
+	var (
+		buf    bytes.Buffer
+		writer *gzip.Writer
+		pool   *sync.Pool
+	)
+
+	switch level {
+	case CompressionLevelDefault:
+		pool = &gzipWriterPool
+	case 1:
+		pool = &gzipWriterPoolForCompressionLevel1
+	case 2:
+		pool = &gzipWriterPoolForCompressionLevel2
+	case 3:
+		pool = &gzipWriterPoolForCompressionLevel3
+	case 4:
+		pool = &gzipWriterPoolForCompressionLevel4
+	case 5:
+		pool = &gzipWriterPoolForCompressionLevel5
+	case 6:
+		pool = &gzipWriterPoolForCompressionLevel6
+	case 7:
+		pool = &gzipWriterPoolForCompressionLevel7
+	case 8:
+		pool = &gzipWriterPoolForCompressionLevel8
+	case 9:
+		pool = &gzipWriterPoolForCompressionLevel9
+	default:
+		var err error
+		writer, err = gzip.NewWriterLevel(&buf, level)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if pool != nil {
+		writer = pool.Get().(*gzip.Writer)
+		writer.Reset(&buf)
+		defer pool.Put(writer)
+	}
+	if _, err := writer.Write(data); err != nil {
+		return nil, err
+	}
+	if err := writer.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func lz4Compress(data []byte) ([]byte, error) {
+	writer := lz4WriterPool.Get().(*lz4.Writer)
+	defer lz4WriterPool.Put(writer)
+
+	var buf bytes.Buffer
+	writer.Reset(&buf)
+	if _, err := writer.Write(data); err != nil {
+		return nil, err
+	}
+	if err := writer.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}


Test patch:
diff --git a/message_test.go b/message_test.go
index 16cfec5e2..bcda158d8 100644
--- a/message_test.go
+++ b/message_test.go
@@ -46,15 +46,15 @@ var (
 	}
 
 	emptyLZ4Message = []byte{
-		132, 219, 238, 101, // CRC
+		136, 42, 245, 190, // CRC
 		0x01,                          // version byte
 		0x03,                          // attribute flags: lz4
 		0, 0, 1, 88, 141, 205, 89, 56, // timestamp
 		0xFF, 0xFF, 0xFF, 0xFF, // key
 		0x00, 0x00, 0x00, 0x0f, // len
 		0x04, 0x22, 0x4D, 0x18, // LZ4 magic number
-		100,                  // LZ4 flags: version 01, block independent, content checksum
-		112, 185, 0, 0, 0, 0, // LZ4 data
+		100,                 // LZ4 flags: version 01, block independent, content checksum
+		64, 167, 0, 0, 0, 0, // LZ4 data
 		5, 93, 204, 2, // LZ4 checksum
 	}
 
diff --git a/record_test.go b/record_test.go
index b827367c2..02c594d1e 100644
--- a/record_test.go
+++ b/record_test.go
@@ -218,7 +218,7 @@ func recordBatchTestCases() []struct {
 				0, 0, 0, 89, // Length
 				0, 0, 0, 0, // Partition Leader Epoch
 				2,                 // Version
-				169, 74, 119, 197, // CRC
+				157, 18, 145, 248, // CRC
 				0, 3, // Attributes
 				0, 0, 0, 0, // Last Offset Delta
 				0, 0, 1, 88, 141, 205, 89, 56, // First Timestamp
@@ -227,10 +227,11 @@ func recordBatchTestCases() []struct {
 				0, 0, // Producer Epoch
 				0, 0, 0, 0, // First Sequence
 				0, 0, 0, 1, // Number of Records
-				4, 34, 77, 24, 100, 112, 185, 21, 0, 0, 128, 40, 0, 10, 0, 8, 1, 2, 3, 4, 6, 5, 6, 7, 2,
+				4, 34, 77, 24, 100, 64, 167, 21, 0, 0, 128, 40, 0, 10, 0, 8, 1, 2, 3, 4, 6, 5, 6, 7, 2,
 				6, 8, 9, 10, 4, 11, 12, 0, 0, 0, 0, 12, 59, 239, 146,
 			},
 		},
+		//
 	}
 }