Repository: scality/cloudserver
Pull request #5757: CLDSRV-620: Ignore trailing checksums in upload requests
Author: login: fredmnl
Created: 2025-03-07T14:19:36Z | Merged: 2025-03-17T20:22:35Z

# Motivation

Since aws-cli 2.23 (and associated SDK version), put-object now defaults to using unsigned payload with trailing checksums for integrity.

This is particularly nice from a computational point of view because the client only needs to read their file once to perform the upload. With AuthV4 signed payload, you would need to first compute the checksum of the object, to eventually include it in the signature contained in the header. With trailing checksums (and unsigned payload), the checksum is computed while the upload is happening and the checksum is sent at the end of the body. 

# Implementation

In this first iteration we are only interested in accepting the incoming requests using trailing checksums and unsigned payload. We are not yet verifying the checksum although the current proposal is extensible to perform this checksum verification as well. 






Diff:
diff --git a/constants.js b/constants.js
index 09cbe799c0..c547c44dd6 100644
--- a/constants.js
+++ b/constants.js
@@ -187,13 +187,13 @@ const constants = {
     // Session name of the backbeat lifecycle assumed role session.
     backbeatLifecycleSessionName: 'backbeat-lifecycle',
     unsupportedSignatureChecksums: new Set([
-        'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
         'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
         'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
         'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
     ]),
     supportedSignatureChecksums: new Set([
         'UNSIGNED-PAYLOAD',
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
         'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
     ]),
     ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js
index 7d436dd96b..493e25f185 100644
--- a/lib/api/apiUtils/object/prepareStream.js
+++ b/lib/api/apiUtils/object/prepareStream.js
@@ -1,4 +1,5 @@
 const V4Transform = require('../../../auth/streamingV4/V4Transform');
+const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform');
 
 /**
  * Prepares the stream if the chunks are sent in a v4 Auth request
@@ -24,11 +25,25 @@ function prepareStream(stream, streamingV4Params, log, errCb) {
         }
         const v4Transform = new V4Transform(streamingV4Params, log, errCb);
         stream.pipe(v4Transform);
+        v4Transform.headers = stream.headers;
         return v4Transform;
     }
     return stream;
 }
 
+function stripTrailingChecksumStream(stream, log) {
+    // don't do anything if we are not in the correct integrity check mode
+    if (stream.headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
+        return stream;
+    }
+
+    const trailingChecksumTransform = new TrailingChecksumTransform(log);
+    stream.pipe(trailingChecksumTransform);
+    trailingChecksumTransform.headers = stream.headers;
+    return trailingChecksumTransform;
+}
+
 module.exports = {
     prepareStream,
+    stripTrailingChecksumStream,
 };
diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js
index 346683e66c..8beea03ecb 100644
--- a/lib/api/apiUtils/object/storeObject.js
+++ b/lib/api/apiUtils/object/storeObject.js
@@ -1,7 +1,7 @@
 const { errors, jsutil } = require('arsenal');
 
 const { data } = require('../../../data/wrapper');
-const { prepareStream } = require('./prepareStream');
+const { prepareStream, stripTrailingChecksumStream } = require('./prepareStream');
 
 /**
  * Check that `hashedStream.completedHash` matches header `stream.contentMD5`
@@ -58,10 +58,11 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
 function dataStore(objectContext, cipherBundle, stream, size,
     streamingV4Params, backendInfo, log, cb) {
     const cbOnce = jsutil.once(cb);
-    const dataStream = prepareStream(stream, streamingV4Params, log, cbOnce);
-    if (!dataStream) {
+    const dataStreamTmp = prepareStream(stream, streamingV4Params, log, cbOnce);
+    if (!dataStreamTmp) {
         return process.nextTick(() => cb(errors.InvalidArgument));
     }
+    const dataStream = stripTrailingChecksumStream(dataStreamTmp, log, cbOnce);
     return data.put(
         cipherBundle, dataStream, size, objectContext, backendInfo, log,
         (err, dataRetrievalInfo, hashedStream) => {
diff --git a/lib/api/apiUtils/object/validateChecksumHeaders.js b/lib/api/apiUtils/object/validateChecksumHeaders.js
index 8162603695..90c5041516 100644
--- a/lib/api/apiUtils/object/validateChecksumHeaders.js
+++ b/lib/api/apiUtils/object/validateChecksumHeaders.js
@@ -5,8 +5,10 @@ const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('
 function validateChecksumHeaders(headers) {
     // If the x-amz-trailer header is present the request is using one of the
     // trailing checksum algorithms, which are not supported.
-    if (headers['x-amz-trailer'] !== undefined) {
-        return errors.BadRequest.customizeDescription('trailing checksum is not supported');
+
+    if (headers['x-amz-trailer'] !== undefined &&
+        headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
+        return errors.BadRequest.customizeDescription('signed trailing checksum is not supported');
     }
 
     const signatureChecksum = headers['x-amz-content-sha256'];
diff --git a/lib/auth/streamingV4/V4Transform.js b/lib/auth/streamingV4/V4Transform.js
index 0533d7c7a0..e28d6efcfb 100644
--- a/lib/auth/streamingV4/V4Transform.js
+++ b/lib/auth/streamingV4/V4Transform.js
@@ -184,7 +184,7 @@ class V4Transform extends Transform {
      * @param {Buffer} chunk - chunk from request body
      * @param {string} encoding - Data encoding
      * @param {function} callback - Callback(err, justDataChunk, encoding)
-     * @return {function }executes callback with err if applicable
+     * @return {function} executes callback with err if applicable
      */
     _transform(chunk, encoding, callback) {
         // 'chunk' here is the node streaming chunk
diff --git a/lib/auth/streamingV4/trailingChecksumTransform.js b/lib/auth/streamingV4/trailingChecksumTransform.js
new file mode 100644
index 0000000000..48870c80e7
--- /dev/null
+++ b/lib/auth/streamingV4/trailingChecksumTransform.js
@@ -0,0 +1,119 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+const { maximumAllowedPartSize } = require('../../../constants');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger: the max
+            // chunk size is 5GB (see constants.maximumAllowedPartSize)
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.subarray(0, 11).toString('hex'),
+                    chunkSizeBufferLength: this.chunkSizeBuffer.length,
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
+            }
+            if (lineBreakIndex === -1) {
+                // no delimiter, we'll keep the chunk for later
+                this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk]);
+                return callback();
+            }
+
+            this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk.subarray(0, lineBreakIndex)]);
+            chunk = chunk.subarray(lineBreakIndex);
+
+            // chunk-size is sent in hex
+            const chunkSizeStr = this.chunkSizeBuffer.toString();
+            const dataSize = parseInt(chunkSizeStr, 16);
+            // we check that the parsing is correct (parseInt returns a partial parse when it fails)
+            if (isNaN(dataSize) || dataSize.toString(16) !== chunkSizeStr.toLowerCase()) {
+                this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
+                return callback(errors.InvalidArgument);
+            }
+            this.chunkSizeBuffer = Buffer.alloc(0);
+            if (dataSize === 0) {
+                // TODO: check if the checksum is correct (S3C-9732)
+                // last chunk, no more data to read, the stream is closed
+                this.streamClosed = true;
+            }
+            if (dataSize > maximumAllowedPartSize) {
+                this.log.error('chunk size too big', { dataSize });
+                return callback(errors.EntityTooLarge);
+            }
+            this.bytesToRead = dataSize;
+            this.bytesToDiscard = 2;
+        }
+
+        return callback();
+    }
+}
+
+module.exports = TrailingChecksumTransform;
diff --git a/package.json b/package.json
index 4eb2151298..ecefbae9e3 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "s3",
-  "version": "7.10.52",
+  "version": "7.10.53",
   "description": "S3 connector",
   "main": "index.js",
   "engines": {
diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js
new file mode 100644
index 0000000000..bad429c3c8
--- /dev/null
+++ b/tests/functional/raw-node/test/trailingChecksums.js
@@ -0,0 +1,133 @@
+const assert = require('assert');
+const async = require('async');
+const { makeS3Request } = require('../utils/makeRequest');
+const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4');
+
+const bucket = 'testunsupportedchecksumsbucket';
+const objectKey = 'key';
+const objData = Buffer.alloc(1024, 'a');
+// note this is not the correct checksum in objDataWithTrailingChecksum
+const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' +
+                                    '10\r\n0123456789abcdef\r\n' +
+                                    '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n';
+const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef';
+
+const config = require('../../config.json');
+const authCredentials = {
+    accessKey: config.accessKey,
+    secretKey: config.secretKey,
+};
+
+const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it;
+
+describe('trailing checksum requests:', () => {
+    before(done => {
+        makeS3Request({
+            method: 'PUT',
+            authCredentials,
+            bucket,
+        }, err => {
+            assert.ifError(err);
+            done();
+        });
+    });
+
+    after(done => {
+        async.series([
+            next => makeS3Request({
+                method: 'DELETE',
+                authCredentials,
+                bucket,
+                objectKey,
+            }, next),
+            next => makeS3Request({
+                method: 'DELETE',
+                authCredentials,
+                bucket,
+            }, next),
+        ], err => {
+            assert.ifError(err);
+            done();
+        });
+    });
+
+    it('should accept unsigned trailing checksum', done => {
+        const req = new HttpRequestAuthV4(
+            `http://localhost:8000/${bucket}/${objectKey}`,
+            Object.assign(
+                {
+                    method: 'PUT',
+                    headers: {
+                        'content-length': objDataWithTrailingChecksum.length,
+                        'x-amz-decoded-content-length': objDataWithoutTrailingChecksum.length,
+                        'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
+                        'x-amz-trailer': 'x-amz-checksum-crc64nvme',
+                    },
+                },
+                authCredentials
+            ),
+            res => {
+                assert.strictEqual(res.statusCode, 200);
+                res.on('data', () => {});
+                res.on('end', done);
+            }
+        );
+
+        req.on('error', err => {
+            assert.ifError(err);
+        });
+
+        req.write(objDataWithTrailingChecksum);
+
+        req.once('drain', () => {
+            req.end();
+        });
+    });
+
+    it('should have correct object content for unsigned trailing checksum', done => {
+        makeS3Request({
+            method: 'GET',
+            authCredentials,
+            bucket,
+            objectKey,
+        }, (err, res) => {
+            assert.ifError(err);
+            assert.strictEqual(res.statusCode, 200);
+            // check that the object data is the input stripped of the trailing checksum
+            assert.strictEqual(res.body, objDataWithoutTrailingChecksum);
+            return done();
+        });
+    });
+
+    itSkipIfAWS('should respond with BadRequest for signed trailing checksum', done => {
+        const req = new HttpRequestAuthV4(
+            `http://localhost:8000/${bucket}/${objectKey}`,
+            Object.assign(
+                {
+                    method: 'PUT',
+                    headers: {
+                        'content-length': objData.length,
+                        'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
+                        'x-amz-trailer': 'x-amz-checksum-sha256',
+                    },
+                },
+                authCredentials
+            ),
+            res => {
+                assert.strictEqual(res.statusCode, 400);
+                res.on('data', () => {});
+                res.on('end', done);
+            }
+        );
+
+        req.on('error', err => {
+            assert.ifError(err);
+        });
+
+        req.write(objData);
+
+        req.once('drain', () => {
+            req.end();
+        });
+    });
+});
diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js
new file mode 100644
index 0000000000..9cae54e659
--- /dev/null
+++ b/tests/unit/auth/TrailingChecksumTransform.js
@@ -0,0 +1,206 @@
+const { errors } = require('arsenal');
+const assert = require('assert');
+const async = require('async');
+const { Readable } = require('stream');
+
+const TrailingChecksumTransform = require('../../../lib/auth/streamingV4/trailingChecksumTransform');
+const { DummyRequestLogger } = require('../helpers');
+
+const log = new DummyRequestLogger();
+
+// note this is not the correct checksum in objDataWithTrailingChecksum
+const objDataWithTrailingChecksum = '10\r\n01234\r6789abcd\r\n\r\n' +
+                                    '2\r\n01\r\n' +
+                                    '1\r\n2\r\n' +
+                                    'd\r\n3456789abcdef\r\n' +
+                                    '0\r\nchecksum:xyz=\r\n';
+const objDataWithoutTrailingChecksum = '01234\r6789abcd\r\n0123456789abcdef';
+
+class ChunkedReader extends Readable {
+    constructor(chunks) {
+        super();
+        this._parts = chunks;
+        this._index = 0;
+    }
+
+    _read() {
+        if (this._index >= this._parts.length) {
+            this.push(null);
+            return;
+        }
+        this.push(this._parts[this._index]);
+        this._index++;
+    }
+
+    getIndex() {
+        return this._index;
+    }
+}
+
+describe('TrailingChecksumTransform class', () => {
+    it('should correctly remove checksums', done => {
+        const trailingChecksumTransform = new TrailingChecksumTransform(log);
+        trailingChecksumTransform.on('error', err => {
+            assert.strictEqual(err, null);
+        });
+        const chunks = [
+            Buffer.from(objDataWithTrailingChecksum),
+        ];
+        const chunkedReader = new ChunkedReader(chunks);
+        chunkedReader.pipe(trailingChecksumTransform);
+        const outputChunks = [];
+        trailingChecksumTransform.on('data', chunk => outputChunks.push(Buffer.from(chunk)));
+        trailingChecksumTransform.on('finish', () => {
+            const data = Buffer.concat(outputChunks).toString();
+            assert.strictEqual(data, objDataWithoutTrailingChecksum);
+            done();
+        });
+        trailingChecksumTransform.on('error', err => {
+            assert.ifError(err);
+        });
+    });
+
+    // test all bisection of the input string
+    async.forEach([...Array(objDataWithTrailingChecksum.length).keys()], i => {
+        it(`should correctly remove checksums, cut at ${i}`, done => {
+            const trailingChecksumTransform = new TrailingChecksumTransform(log);
+            trailingChecksumTransform.on('error', err => {
+                assert.strictEqual(err, null);
+            });
+            const chunks = [
+                Buffer.from(objDataWithTrailingChecksum.substring(0, i)),
+                Buffer.from(objDataWithTrailingChecksum.substring(i)),
+            ];
+            const chunkedReader = new ChunkedReader(chunks);
+            chunkedReader.pipe(trailingChecksumTransform);
+            const outputChunks = [];
+            trailingChecksumTransform.on('data', chunk => outputChunks.push(Buffer.from(chunk)));
+            trailingChecksumTransform.on('finish', () => {
+                const data = Buffer.concat(outputChunks).toString();
+                assert.strictEqual(data, objDataWithoutTrailingChecksum);
+                done();
+            });
+            trailingChecksumTransform.on('error', err => {
+                assert.ifError(err);
+            });
+        });
+    });
+
+    // test all trisection of the input string
+    async.forEach([...Array(objDataWithTrailingChecksum.length - 2).keys()], i => {
+        async.forEach([...Array(objDataWithTrailingChecksum.length - i - 2).keys()], j => {
+            it(`should correctly remove checksums, cut at ${i} and ${i + j + 1}`, done => {
+                const trailingChecksumTransform = new TrailingChecksumTransform(log);
+                trailingChecksumTransform.on('error', err => {
+                    assert.strictEqual(err, null);
+                });
+                const chunks = [
+                    Buffer.from(objDataWithTrailingChecksum.substring(0, i + 1)),
+                    Buffer.from(objDataWithTrailingChecksum.substring(i + 1, i + j + 2)),
+                    Buffer.from(objDataWithTrailingChecksum.substring(i + j + 2)),
+                ];
+                const chunkedReader = new ChunkedReader(chunks);
+                chunkedReader.pipe(trailingChecksumTransform);
+                const outputChunks = [];
+                trailingChecksumTransform.on('data', chunk => outputChunks.push(Buffer.from(chunk)));
+                trailingChecksumTransform.on('finish', () => {
+                    const data = Buffer.concat(outputChunks).toString();
+                    assert.strictEqual(data, objDataWithoutTrailingChecksum);
+                    done();
+                });
+                trailingChecksumTransform.on('error', err => {
+                    assert.ifError(err);
+                });
+            });
+        });
+    });
+
+    it('should correctly remove checksums, cut at each individual byte', done => {
+        const trailingChecksumTransform = new TrailingChecksumTransform(log);
+        trailingChecksumTransform.on('error', err => {
+            assert.strictEqual(err, null);
+        });
+        const chunks = [];
+        for (let i = 0; i < objDataWithTrailingChecksum.length; i++) {
+            chunks.push(objDataWithTrailingChecksum.substring(i, i + 1));
+        }
+        const chunkedReader = new ChunkedReader(chunks);
+        chunkedReader.pipe(trailingChecksumTransform);
+        const outputChunks = [];
+        trailingChecksumTransform.on('data', chunk => outputChunks.push(Buffer.from(chunk)));
+        trailingChecksumTransform.on('finish', () => {
+            const data = Buffer.concat(outputChunks).toString();
+            assert.strictEqual(data, objDataWithoutTrailingChecksum);
+            done();
+        });
+        trailingChecksumTransform.on('error', err => {
+            assert.ifError(err);
+        });
+    });
+
+    it('should return an error if the format does not follow trailing checksum specification', done => {
+        const trailingChecksumTransform = new TrailingChecksumTransform(log);
+        const chunks = [
+            Buffer.from('11\r\n'),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+        ];
+        const chunkedReader = new ChunkedReader(chunks);
+        trailingChecksumTransform.on('error', err => {
+            assert.deepStrictEqual(err, errors.InvalidArgument);
+            trailingChecksumTransform.end();
+        });
+        let bytesWritten = 0;
+        trailingChecksumTransform.on('data', chunk => {
+            bytesWritten += chunk.length;
+        });
+        trailingChecksumTransform.on('close', () => {
+            assert.equal(bytesWritten, 17);
+            // 2 is the minimum but it looks like buffering will fetch additional chunks before the error is emitted
+            // as long as we do not read the full input stream, this is fine
+            assert.ok(chunkedReader.getIndex() <= 4);
+            done();
+        });
+        chunkedReader.pipe(trailingChecksumTransform);
+    });
+
+    it('should return early if supplied with an out of specification chunk size', done => {
+        const trailingChecksumTransform = new TrailingChecksumTransform(log);
+        const chunks = [
+            Buffer.from('500000'),
+            Buffer.from('000000\r\n'),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+            Buffer.alloc(1000000),
+        ];
+        const chunkedReader = new ChunkedReader(chunks);
+        trailingChecksumTransform.on('error', err => {
+            assert.deepStrictEqual(err, errors.InvalidArgument);
+            trailingChecksumTransform.end();
+        });
+        let bytesWritten = 0;
+        trailingChecksumTransform.on('data', chunk => {
+            bytesWritten += chunk.length;
+        });
+        trailingChecksumTransform.on('close', () => {
+            assert.equal(bytesWritten, 0);
+            // 2 is the minimum but it looks like buffering will fetch additional chunks before the error is emitted
+            // as long as we do not read the full input stream, this is fine
+            assert.ok(chunkedReader.getIndex() <= 4);
+            done();
+        });
+        chunkedReader.pipe(trailingChecksumTransform);
+    });
+});


Reviews:
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-11T08:59:42Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-11T09:36:58Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-11T09:55:38Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-11T09:57:06Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-11T10:10:48Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T13:29:28Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T20:08:44Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T20:09:36Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T20:18:29Z
user: login: tcarmet
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T16:33:08Z
user: login: tmacro
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T16:55:12Z
user: login: tmacro
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T16:55:42Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T17:25:47Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-17T08:08:41Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T09:20:06Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-17T09:27:38Z
user: login: fredmnl
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T10:56:21Z
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-17T11:55:43Z
state: APPROVED
author_association: NONE
submitted_at: 2025-03-17T11:56:48Z
body: Just a comment so we don't forget to bump cloudserver's version before merging, as we need it for the next release(s) :slightly_smiling_face: 
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-17T11:57:30Z

Review comments:
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: `\r` alone is not the documented CRLF (`\r\n`), and might be present in the data alone (as the buffer might have any data, unless we confirm this is not possible?). We must not split based on this character only. For instance the following unit test should pass, but it returns an `InvalidArgument` error:

```js
    it.only('should handle CR in data properly with fixed implementation', done => {
        const trailingChecksumTransform = new TrailingChecksumTransform(log);
        const chunks = [
            // First chunk with size and CRLF delimiter
            Buffer.from('5\r\n'),
            // Second chunk with data containing a CR character
            Buffer.from('a\rb\nc'),
            // Third chunk with CRLF after data, then next chunk
            Buffer.from('\r\n5\r\nhello\r\n0\r\nchecksum:xyz=\r\n')
        ];
        
        const chunkedReader = new ChunkedReader(chunks);
        chunkedReader.pipe(trailingChecksumTransform);
        const outputChunks = [];
        
        trailingChecksumTransform.on('data', chunk => {
            console.log('Data:', chunk.toString());
            outputChunks.push(Buffer.from(chunk));
        });
        
        trailingChecksumTransform.on('finish', () => {
            const data = Buffer.concat(outputChunks).toString();
            console.log('Final Output:', data);
            assert.strictEqual(data, 'a\rb\nc');
            done();
        });
        
        trailingChecksumTransform.on('error', err => {
            console.log('Error:', err);
            done(err);
        });
    });
```
created_at: 2025-03-11T08:48:04Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: If you need to handle errors in this class, you may implement the `_flush` method, something like

```js
_flush(callback) {
    if (!this.streamClosed) {
        this.log.error('stream ended without closing chunked encoding');
        return callback(errors.InvalidArgument);
    }
    callback();
}
```
created_at: 2025-03-11T08:59:02Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Worth adding a comment to explain that we are not validating the checksums, and why
created_at: 2025-03-11T09:35:55Z
author_association: NONE
diff_hunk: @@ -24,11 +25,26 @@ function prepareStream(stream, streamingV4Params, log, errCb) {
         }
         const v4Transform = new V4Transform(streamingV4Params, log, errCb);
         stream.pipe(v4Transform);
+        v4Transform.headers = stream.headers;
         return v4Transform;
     }
     return stream;
 }
 
+function stripTrailingChecksumStream(stream, log) {
+    // don't do anything if we are not in the correct integrity check mode
+    if (stream.headers['x-amz-content-sha256'] !==
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
+        return stream;
+    }
+
+    const trailingChecksumTransform = new TrailingChecksumTransform(log);
+    stream.pipe(trailingChecksumTransform);
+    trailingChecksumTransform.headers = stream.headers;
+    return trailingChecksumTransform;
path: lib/api/apiUtils/object/prepareStream.js
body: Good to confirm as well if we are still enforcing the common md5 check in this mode (both stored in the object md and that we properly fail in case of mismatch). Could be through a new unit/ft test.
created_at: 2025-03-11T09:36:56Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Closing as discussed as it works as expected
created_at: 2025-03-11T09:55:38Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
+            if (lineBreakIndex === -1) {
+                if (this.chunkSizeBuffer.byteLength + chunk.byteLength > 10) {
+                    this.log.error('chunk size field too big', {
+                        chunkSizeBuffer: this.chunkSizeBuffer.toString(),
+                        truncatedChunk: chunk.subarray(0, 16).toString(),
+                    });
+                    // if bigger, the chunk would be over 5 GB
+                    // returning early to avoid a DoS by memory exhaustion
+                    return callback(errors.InvalidArgument);
+                }
+                // no delimiter, we'll keep the chunk for later
+                this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk]);
+                return callback();
+            }
+
+            this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk.subarray(0, lineBreakIndex)]);
+            chunk = chunk.subarray(lineBreakIndex);
+
+            // chunk-size is sent in hex
+            if (!/^[0-9a-fA-F]+$/.test(this.chunkSizeBuffer.toString())) {
+                this.log.error('chunk size is not a valid hex number', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString(),
+                });
+                return callback(errors.InvalidArgument);
+            }
+            const dataSize = Number.parseInt(this.chunkSizeBuffer.toString(), 16);
+            if (Number.isNaN(dataSize)) {
+                this.log.error('unable to parse chunk size', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString(),
+                });
+                return callback(errors.InvalidArgument);
+            }
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: ```suggestion
            const chunkSizeStr = this.chunkSizeBuffer.toString();
            const dataSize = parseInt(chunkSizeStr, 16);
            if (isNaN(dataSize) || dataSize.toString(16) !== chunkSizeStr.toLowerCase()) {
                  this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
                  return callback(errors.InvalidArgument);
            }
```
created_at: 2025-03-11T09:57:05Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
+            if (lineBreakIndex === -1) {
+                if (this.chunkSizeBuffer.byteLength + chunk.byteLength > 10) {
+                    this.log.error('chunk size field too big', {
+                        chunkSizeBuffer: this.chunkSizeBuffer.toString(),
+                        truncatedChunk: chunk.subarray(0, 16).toString(),
+                    });
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Is there a risk of DoS here if we log the actual data sent? If we send a very long number, like, very big, shall we just log the length and not the actual buffer?
Same if the buffer has invalid characters, we might break the json-valid logs.

Same comment in other places when applicable.
created_at: 2025-03-11T10:05:43Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
+            if (lineBreakIndex === -1) {
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: The length should be 10 bytes max, according to this function path, but if it's part of the same chunk (so NOT entering this `if`), I am able to send a very large number, like `500000000000000000000\r\n`; is that expected?
created_at: 2025-03-11T10:10:28Z
author_association: NONE
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
+            if (lineBreakIndex === -1) {
+                if (this.chunkSizeBuffer.byteLength + chunk.byteLength > 10) {
+                    this.log.error('chunk size field too big', {
+                        chunkSizeBuffer: this.chunkSizeBuffer.toString(),
+                        truncatedChunk: chunk.subarray(0, 16).toString(),
+                    });
path: lib/auth/streamingV4/trailingChecksumTransform.js
user: login: fredmnl
body: You're right, we were not enforcing the length restriction when the big field is found within the chunk. Fixing it. 
created_at: 2025-03-11T13:29:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -24,11 +25,26 @@ function prepareStream(stream, streamingV4Params, log, errCb) {
         }
         const v4Transform = new V4Transform(streamingV4Params, log, errCb);
         stream.pipe(v4Transform);
+        v4Transform.headers = stream.headers;
         return v4Transform;
     }
     return stream;
 }
 
+function stripTrailingChecksumStream(stream, log) {
+    // don't do anything if we are not in the correct integrity check mode
+    if (stream.headers['x-amz-content-sha256'] !==
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
+        return stream;
+    }
+
+    const trailingChecksumTransform = new TrailingChecksumTransform(log);
+    stream.pipe(trailingChecksumTransform);
+    trailingChecksumTransform.headers = stream.headers;
+    return trailingChecksumTransform;
path: lib/api/apiUtils/object/prepareStream.js
user: login: fredmnl
body: So I checked and when the trailing checksum mode is activated, the client does not send `md5` checksums. Therefore in this mode we do not check integrity. This is not a first, there are already ways to upload where `cloudserver` does not check payload integrity. In fact, it only checks payload integrity in two scenarios: `content-md5` header is provided, or streaming AuthV4 is used (in this case, the signature verification also indirectly verifies payload integrity).

In the absence of integrity check in cloudserver, integrity is still verified at the TCP level (although it's quite a weak verification) and at the HTTPS level. 

I have circled back with David to triple check that this is what we want from a product standpoint.


created_at: 2025-03-11T20:08:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            const lineBreakIndex = chunk.indexOf('\r');
+            if (lineBreakIndex === -1) {
path: lib/auth/streamingV4/trailingChecksumTransform.js
user: login: fredmnl
body: Correct, I fixed it, thanks :+1: 
created_at: 2025-03-11T20:09:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,98 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * v4 Auth request
+ */
+class TrailingChecksumTransform extends Transform {
path: lib/auth/streamingV4/trailingChecksumTransform.js
user: login: fredmnl
body: That's great! :+1: 
created_at: 2025-03-11T20:18:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,70 +0,0 @@
-const assert = require('assert');
path: tests/functional/raw-node/test/unsupportedChecksums.js
user: login: tmacro
body: Unless an upcoming PR will be landing support for `STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER` I think we should keep this test case
created_at: 2025-03-13T16:55:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,70 +0,0 @@
-const assert = require('assert');
path: tests/functional/raw-node/test/unsupportedChecksums.js
user: login: fredmnl
body: Absolutely, not sure why I did that, probably got confused with the different headers. Good catch :+1: 

created_at: 2025-03-13T17:25:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -24,11 +25,26 @@ function prepareStream(stream, streamingV4Params, log, errCb) {
         }
         const v4Transform = new V4Transform(streamingV4Params, log, errCb);
         stream.pipe(v4Transform);
+        v4Transform.headers = stream.headers;
         return v4Transform;
     }
     return stream;
 }
 
+function stripTrailingChecksumStream(stream, log) {
+    // don't do anything if we are not in the correct integrity check mode
+    if (stream.headers['x-amz-content-sha256'] !==
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
+        return stream;
+    }
+
+    const trailingChecksumTransform = new TrailingChecksumTransform(log);
+    stream.pipe(trailingChecksumTransform);
+    trailingChecksumTransform.headers = stream.headers;
+    return trailingChecksumTransform;
path: lib/api/apiUtils/object/prepareStream.js
body: I see, if we all align, good to mention it in the release notes: integrity is not guaranteed by the S3 service with trailing checksums PUTs.
created_at: 2025-03-17T07:17:02Z
author_association: NONE
diff_hunk: @@ -24,11 +25,26 @@ function prepareStream(stream, streamingV4Params, log, errCb) {
         }
         const v4Transform = new V4Transform(streamingV4Params, log, errCb);
         stream.pipe(v4Transform);
+        v4Transform.headers = stream.headers;
         return v4Transform;
     }
     return stream;
 }
 
+function stripTrailingChecksumStream(stream, log) {
+    // don't do anything if we are not in the correct integrity check mode
+    if (stream.headers['x-amz-content-sha256'] !==
+        'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
path: lib/api/apiUtils/object/prepareStream.js
body: ```suggestion
    if (stream.headers['x-amz-content-sha256'] !== `STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
```
created_at: 2025-03-17T07:19:45Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: ```suggestion
            // no need to look further than 10 bytes since the field cannot be bigger: the max
            // chunk size is 5GB (see constants.maximumAllowedPartSize)
```
created_at: 2025-03-17T07:38:59Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: To return a standard error (see `lib/api/objectPutPart.js`) :

```suggestion
                return callback(errors.EntityTooLarge);
```
created_at: 2025-03-17T07:39:10Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Better not to log fully the chunk if its size exceeds 10, as we then don't know how big it is. We may log one more byte and the total length to have some info.

```suggestion
                    chunkSizeBuffer: this.chunkSizeBuffer.subarray(0, 11).toString('hex'),
                    chunkSizeBufferLength: this.chunkSizeBuffer.length,
                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
```
created_at: 2025-03-17T07:48:27Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
+            }
+            if (lineBreakIndex === -1) {
+                // no delimiter, we'll keep the chunk for later
+                this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk]);
+                return callback();
+            }
+
+            this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk.subarray(0, lineBreakIndex)]);
+            chunk = chunk.subarray(lineBreakIndex);
+
+            // chunk-size is sent in hex
+            const chunkSizeStr = this.chunkSizeBuffer.toString();
+            const dataSize = parseInt(chunkSizeStr, 16);
+            // we check that the parsing is correct (parseInt returns a partial parse when it fails)
+            if (isNaN(dataSize) || dataSize.toString(16) !== chunkSizeStr.toLowerCase()) {
+                this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
+                return callback(errors.InvalidArgument);
+            }
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: I re-thought about it and I found that with this approach we cannot support leading zeros (like in `000019`). I am not sure this is a valid way to send the values, but leading zeros are common practice with hexa...

Here is a regex-free approach to support it (if needed!):

```suggestion
            const dataSize = parseInt(chunkSizeStr, 16);
            if (isNaN(dataSize)) {
                this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
                return callback(errors.InvalidArgument);
            }
            const hexSize = dataSize.toString(16).toLowerCase();
            let leadingZeros = 0;
            while (leadingZeros < chunkSizeStr.length && chunkSizeStr[leadingZeros] === '0') {
                leadingZeros++;
            }
            if (dataSize === 0) {
                if (leadingZeros !== chunkSizeStr.length) {
                    this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
                    return callback(errors.InvalidArgument);
                }
            } else if (hexSize.length + leadingZeros !== chunkSizeStr.length) {
                this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
                return callback(errors.InvalidArgument);
            }
```

Here is a test that can trigger the problem:

```js
    it('should handle CR in data properly with leading zeros', done => {
        const trailingChecksumTransform = new TrailingChecksumTransform(log);
        const chunks = [
            Buffer.from('00000005\r\n'),         // Chunk size 5 with leading zeros
            Buffer.from('a\nb\nc'),             // 5 bytes of data: 'a', '\n', 'b', '\n', 'c'
            Buffer.from('\r\n0\r\nchecksum:xyz=\r\n') // CRLF, zero chunk, trailer
        ];
    
        const chunkedReader = new ChunkedReader(chunks);
        chunkedReader.pipe(trailingChecksumTransform);
        const outputChunks = [];
    
        trailingChecksumTransform.on('data', chunk => {
            console.log('Data:', chunk.toString());
            outputChunks.push(Buffer.from(chunk));
        });
    
        trailingChecksumTransform.on('finish', () => {
            const data = Buffer.concat(outputChunks).toString();
            console.log('Final Output:', data);
            assert.strictEqual(data, 'a\nb\nc');
            done();
        });
    
        trailingChecksumTransform.on('error', err => {
            console.log('Error:', err);
            done(err);
        });
    });
```
created_at: 2025-03-17T08:03:58Z
author_association: NONE
diff_hunk: @@ -0,0 +1,132 @@
+const assert = require('assert');
+const async = require('async');
+const { makeS3Request } = require('../utils/makeRequest');
+const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4');
+
+const bucket = 'testunsupportedchecksumsbucket';
+const objectKey = 'key';
+const objData = Buffer.alloc(1024, 'a');
+// note this is not the correct checksum in objDataWithTrailingChecksum
+const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' +
+                                    '10\r\n0123456789abcdef\r\n' +
+                                    '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n';
+const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef';
+
+const authCredentials = {
+    accessKey: 'accessKey1',
+    secretKey: 'verySecretKey1',
+};
path: tests/functional/raw-node/test/trailingChecksums.js
body: This should be available through `tests/functional/aws-node-sdk/test/support/awsConfig.js`
created_at: 2025-03-17T08:06:29Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Should we not only check the number of bytes in the buffer, but also the actual value, and ensure we don't exceed the max supported (5GB)? See `constants.maximumAllowedPartSize` for the constant you can use.
created_at: 2025-03-17T08:08:37Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
+            }
+            if (lineBreakIndex === -1) {
+                // no delimiter, we'll keep the chunk for later
+                this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk]);
+                return callback();
+            }
+
+            this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk.subarray(0, lineBreakIndex)]);
+            chunk = chunk.subarray(lineBreakIndex);
+
+            // chunk-size is sent in hex
+            const chunkSizeStr = this.chunkSizeBuffer.toString();
+            const dataSize = parseInt(chunkSizeStr, 16);
+            // we check that the parsing is correct (parseInt returns a partial parse when it fails)
+            if (isNaN(dataSize) || dataSize.toString(16) !== chunkSizeStr.toLowerCase()) {
+                this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
+                return callback(errors.InvalidArgument);
+            }
path: lib/auth/streamingV4/trailingChecksumTransform.js
user: login: fredmnl
body: I thought about that as well but decided against it as the AWS doc, the `aws-cli` implementation and the Wikipedia article on chunked transfer ([https://en.wikipedia.org/wiki/Chunked_transfer_encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding)) all do not use leading zeros. I think we can forgo the additional bloat.
created_at: 2025-03-17T09:20:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
+            }
+            if (lineBreakIndex === -1) {
+                // no delimiter, we'll keep the chunk for later
+                this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk]);
+                return callback();
+            }
+
+            this.chunkSizeBuffer = Buffer.concat([this.chunkSizeBuffer, chunk.subarray(0, lineBreakIndex)]);
+            chunk = chunk.subarray(lineBreakIndex);
+
+            // chunk-size is sent in hex
+            const chunkSizeStr = this.chunkSizeBuffer.toString();
+            const dataSize = parseInt(chunkSizeStr, 16);
+            // we check that the parsing is correct (parseInt returns a partial parse when it fails)
+            if (isNaN(dataSize) || dataSize.toString(16) !== chunkSizeStr.toLowerCase()) {
+                this.log.error('invalid chunk size', { chunkSizeBuffer: chunkSizeStr });
+                return callback(errors.InvalidArgument);
+            }
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Sure, at least we now about it and we can improve if a client were to use it.
created_at: 2025-03-17T09:27:37Z
author_association: NONE
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
path: lib/auth/streamingV4/trailingChecksumTransform.js
user: login: fredmnl
body: I'd rather leave it as `Invalid` because we do not actually check what we read at this point. If the client sends garbage, he should receive `Invalid` rather than `EntityTooLarge`. Furthermore I don't think that `Invalid` is incorrect, it's just not necessarily that helpful which I don't think is so important here. 
created_at: 2025-03-17T10:56:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,112 @@
+const { Transform } = require('stream');
+const { errors } = require('arsenal');
+
+/**
+ * This class is designed to handle the chunks sent in a streaming
+ * unsigned playload trailer request. In this iteration, we are not checking
+ * the checksums, but we are removing them from the stream.
+ * S3C-9732 will deal with checksum verification.
+ */
+class TrailingChecksumTransform extends Transform {
+    /**
+     * @constructor
+     * @param {object} log - logger object
+     */
+    constructor(log) {
+        super({});
+        this.log = log;
+        this.chunkSizeBuffer = Buffer.alloc(0);
+        this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
+        this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
+        this.streamClosed = false;
+    }
+
+    /**
+     * This function is executed when there is no more data to be read but before the stream is closed
+     * We will verify that the trailing checksum structure was upheld
+     *
+     * @param {function} callback - Callback(err, data)
+     * @return {function} executes callback with err if applicable
+     */
+    _flush(callback) {
+        if (!this.streamClosed) {
+            this.log.error('stream ended without closing chunked encoding');
+            return callback(errors.InvalidArgument);
+        }
+        return callback();
+    }
+
+    /**
+     * This function will remove the trailing checksum from the stream
+     *
+     * @param {Buffer} chunkInput - chunk from request body
+     * @param {string} encoding - Data encoding
+     * @param {function} callback - Callback(err, justDataChunk, encoding)
+     * @return {function} executes callback with err if applicable
+     */
+    _transform(chunkInput, encoding, callback) {
+        let chunk = chunkInput;
+        while (chunk.byteLength > 0 && !this.streamClosed) {
+            if (this.bytesToDiscard > 0) {
+                const toDiscard = Math.min(this.bytesToDiscard, chunk.byteLength);
+                chunk = chunk.subarray(toDiscard);
+                this.bytesToDiscard -= toDiscard;
+                continue;
+            }
+            // forward up to bytesToRead bytes from the chunk, restart processing on leftover
+            if (this.bytesToRead > 0) {
+                const toRead = Math.min(this.bytesToRead, chunk.byteLength);
+                this.push(chunk.subarray(0, toRead));
+                chunk = chunk.subarray(toRead);
+                this.bytesToRead -= toRead;
+                if (this.bytesToRead === 0) {
+                    this.bytesToDiscard = 2;
+                }
+                continue;
+            }
+
+            // we are now looking for the chunk size field
+            // no need to look further than 10 bytes since the field cannot be bigger
+            const lineBreakIndex = chunk.subarray(0, 10).indexOf('\r');
+            const bytesToKeep = lineBreakIndex === -1 ? chunk.byteLength : lineBreakIndex;
+            if (this.chunkSizeBuffer.byteLength + bytesToKeep > 10) {
+                this.log.error('chunk size field too big', {
+                    chunkSizeBuffer: this.chunkSizeBuffer.toString('hex'),
+                    truncatedChunk: chunk.subarray(0, 10).toString('hex'),
+                });
+                // if bigger, the chunk would be over 5 GB
+                // returning early to avoid a DoS by memory exhaustion
+                return callback(errors.InvalidArgument);
path: lib/auth/streamingV4/trailingChecksumTransform.js
body: Yes ok, maybe we can customize the error description if needed. I'm closing this comment
created_at: 2025-03-17T11:55:43Z
author_association: NONE

PR comments:
user: login: bert-e
created_at: 2025-03-07T14:19:39Z
body: # Hello fredmnl,



My role is to assist you with the merge of this
pull request. Please type `@bert-e help` to get information
on this process, or consult the [user documentation](
https://bert-e.scality.net/scality/cloudserver/doc/user).


<details>
  <summary><b>Available options</b></summary>

  name   | description  | privileged   | authored
  ------ | ------------ | ------------ |----------
  `/after_pull_request` | Wait for the given pull request id to be merged before continuing with the current one. |  | 
  `/bypass_author_approval` | Bypass the pull request author's approval |  :star:  | 
  `/bypass_build_status` | Bypass the build and test status |  :star:  | 
  `/bypass_commit_size` | Bypass the check on the size of the changeset ```TBA``` |  :star:  | 
  `/bypass_incompatible_branch` | Bypass the check on the source branch prefix |  :star:  | 
  `/bypass_jira_check` | Bypass the Jira issue check |  :star:  | 
  `/bypass_peer_approval` | Bypass the pull request peers' approval |  :star:  | 
  `/bypass_leader_approval` | Bypass the pull request leaders' approval |  :star:  | 
  `/approve` | Instruct Bert-E that the author has approved the pull request. |  |  :writing_hand: 
  `/create_pull_requests` | Allow the creation of integration pull requests. |  | 
  `/create_integration_branches` | Allow the creation of integration branches. |  | 
  `/no_octopus` | Prevent Wall-E from doing any octopus merge and use multiple consecutive merge instead |  | 
  `/unanimity` | Change review acceptance criteria from `one reviewer at least` to `all reviewers` |  | 
  `/wait` | Instruct Bert-E not to run until further notice. |  | 
  

</details>



<details>
  <summary><b>Available commands</b></summary>

  name   | description  | privileged
  ------ | ------------ | ------------
  `/help` | Print Bert-E's manual in the pull request. | 
  `/status` | Print Bert-E's current status in the pull request ```TBA``` | 
  `/clear` | Remove all comments from Bert-E from the history ```TBA``` | 
  `/retry` | Re-start a fresh build ```TBA``` | 
  `/build` | Re-start a fresh build ```TBA``` | 
  `/force_reset` | Delete integration branches & pull requests, and restart merge process from the beginning. | 
  `/reset` | Try to remove integration branches unless there are commits on them which do not appear on the source branch. | 
  
</details>




*Status report is not available.*







author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:19:46Z
body: # Request integration branches



Waiting for integration branch creation to be requested by the user.

To request integration branches, please comment on this pull request with the following command:

```
/create_integration_branches
```

Alternatively, the `/approve` and `/create_pull_requests` commands will automatically
create the integration branches.






author_association: CONTRIBUTOR
user: login: fredmnl
created_at: 2025-03-07T14:20:12Z
body: /create_integration_branches
author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:20:25Z
body: # Integration data created



I have created the integration data for the additional destination branches.

* this pull request will merge `bugfix/CLDSRV-620/strip-trailing-checksums` into
`development/7.10`
* [w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/7.70...w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums) will be merged into `development/7.70` 
* [w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/8.8...w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums) will be merged into `development/8.8` 
* [w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/9.0...w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums) will be merged into `development/9.0` 



The following branches will **NOT** be impacted:

* `development/7.4`




You can set option `create_pull_requests` if you need me to create
**integration pull requests** in addition to integration branches, with:

```
@bert-e create_pull_requests
```







*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:20:27Z
body: # Waiting for approval



The following approvals are needed before I can proceed with the merge:


* the author


* 2 peers











*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:30:13Z
body: # History mismatch



Merge commit #e324d8a72894b218324c9f640f175ff54d06cbb0 on the integration branch
`w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` is merging a branch which is neither the current
branch `bugfix/CLDSRV-620/strip-trailing-checksums` nor the development branch
`development/7.70`.

It is likely due to a rebase of the branch `bugfix/CLDSRV-620/strip-trailing-checksums` and the
merge is not possible until all related `w/*` branches are deleted or updated.

**Please use the `reset` command to have me reinitialize these branches.**





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: fredmnl
created_at: 2025-03-07T14:30:24Z
body: /reset
author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:30:33Z
body: # Reset complete



I have successfully deleted this pull request's integration branches.






*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:30:46Z
body: # Integration data created



I have created the integration data for the additional destination branches.

* this pull request will merge `bugfix/CLDSRV-620/strip-trailing-checksums` into
`development/7.10`
* [w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/7.70...w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums) will be merged into `development/7.70` 
* [w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/8.8...w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums) will be merged into `development/8.8` 
* [w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/9.0...w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums) will be merged into `development/9.0` 



The following branches will **NOT** be impacted:

* `development/7.4`




You can set option `create_pull_requests` if you need me to create
**integration pull requests** in addition to integration branches, with:

```
@bert-e create_pull_requests
```







*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:30:48Z
body: # Waiting for approval



The following approvals are needed before I can proceed with the merge:


* the author


* 2 peers











*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:40:23Z
body: # Conflict




A conflict has been raised during the update of
integration branch `w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/7.70`.

Please resolve the conflict on **the integration branch** (`w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums`).


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
 $ git pull  # or "git reset --hard origin/w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums"
 $ git merge origin/development/7.70
 $ # <intense conflict resolution>
 $ git commit
 $ git merge origin/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:45:29Z
body: # Conflict




A conflict has been raised during the update of
integration branch `w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/8.8`.

Please resolve the conflict on **the integration branch** (`w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums`).


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums
 $ git pull  # or "git reset --hard origin/w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums"
 $ git merge origin/development/8.8
 $ # <intense conflict resolution>
 $ git commit
 $ git merge origin/w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-07T14:58:06Z
body: # Waiting for approval



The following approvals are needed before I can proceed with the merge:


* the author


* 2 peers











*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T10:14:48Z
body: # History mismatch



Merge commit #337ebf7afb214667e813a33db351a8e90a11c35a on the integration branch
`w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` is merging a branch which is neither the current
branch `bugfix/CLDSRV-620/strip-trailing-checksums` nor the development branch
`development/7.70`.

It is likely due to a rebase of the branch `bugfix/CLDSRV-620/strip-trailing-checksums` and the
merge is not possible until all related `w/*` branches are deleted or updated.

**Please use the `reset` command to have me reinitialize these branches.**





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T14:16:55Z
body: # History mismatch



Merge commit #a9afd5fa7483e73aa38fa7903a772b7565b978e2 on the integration branch
`w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` is merging a branch which is neither the current
branch `bugfix/CLDSRV-620/strip-trailing-checksums` nor the development branch
`development/7.70`.

It is likely due to a rebase of the branch `bugfix/CLDSRV-620/strip-trailing-checksums` and the
merge is not possible until all related `w/*` branches are deleted or updated.

**Please use the `reset` command to have me reinitialize these branches.**





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: fredmnl
created_at: 2025-03-17T14:17:25Z
body: /force_reset
author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T14:17:35Z
body: # Reset complete



I have successfully deleted this pull request's integration branches.






*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T14:17:46Z
body: # Conflict




A conflict has been raised during the creation of
integration branch `w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/7.70`.

**I have not created the integration branch.**


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout -B w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums origin/development/7.70
 $ git merge origin/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T14:51:52Z
body: # Conflict




A conflict has been raised during the creation of
integration branch `w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/8.8`.

**I have not created the integration branch.**


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout -B w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums origin/development/8.8
 $ git merge origin/w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T14:55:53Z
body: # Conflict




A conflict has been raised during the creation of
integration branch `w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/9.0`.

**I have not created the integration branch.**


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout -B w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums origin/development/9.0
 $ git merge origin/w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T15:03:22Z
body: # Waiting for approval



The following approvals are needed before I can proceed with the merge:


* the author


* 2 peers











*The following options are set:* **create_integration_branches**


author_association: CONTRIBUTOR
user: login: fredmnl
created_at: 2025-03-17T15:10:45Z
body: /approve
author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T15:11:00Z
body: # Build failed



The [build](https://github.com/scality/cloudserver/actions?query=branch%3Aw/9.0/bugfix/CLDSRV-620/strip-trailing-checksums) for [commit](https://github.com/scality/cloudserver/commit/9ef8377ee1a5dd880a4f58a11356163ba31aef80) did not succeed in branch [w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/9.0...w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums)



*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T15:12:22Z
body: # Build failed



The [build](https://github.com/scality/cloudserver/actions?query=branch%3Aw/8.8/bugfix/CLDSRV-620/strip-trailing-checksums) for [commit](https://github.com/scality/cloudserver/commit/da5068ed533009a55711fc51fe5550c5149ad9d9) did not succeed in branch [w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/8.8...w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums)



*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T17:27:55Z
body: # History mismatch



Merge commit #bd46a9a53d8241f0ff9380ea131b239d288edf77 on the integration branch
`w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` is merging a branch which is neither the current
branch `bugfix/CLDSRV-620/strip-trailing-checksums` nor the development branch
`development/7.70`.

It is likely due to a rebase of the branch `bugfix/CLDSRV-620/strip-trailing-checksums` and the
merge is not possible until all related `w/*` branches are deleted or updated.

**Please use the `reset` command to have me reinitialize these branches.**





*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: fredmnl
created_at: 2025-03-17T17:27:55Z
body: /force_reset
author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T17:28:05Z
body: # Reset complete



I have successfully deleted this pull request's integration branches.






*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T17:28:14Z
body: # Conflict




A conflict has been raised during the creation of
integration branch `w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/7.70`.

**I have not created the integration branch.**


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout -B w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums origin/development/7.70
 $ git merge origin/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T17:39:52Z
body: # Conflict




A conflict has been raised during the creation of
integration branch `w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/8.8`.

**I have not created the integration branch.**


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout -B w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums origin/development/8.8
 $ git merge origin/w/7.70/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T17:43:08Z
body: # Conflict




A conflict has been raised during the creation of
integration branch `w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums` with contents from `w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums`
and `development/9.0`.

**I have not created the integration branch.**


Here are the steps to resolve this conflict:

```bash
 $ git fetch
 $ git checkout -B w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums origin/development/9.0
 $ git merge origin/w/8.8/bugfix/CLDSRV-620/strip-trailing-checksums
 $ # <intense conflict resolution>
 $ git commit
 $ git push -u origin w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums
```





*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T17:52:13Z
body: # Build failed



The [build](https://github.com/scality/cloudserver/actions?query=branch%3Aw/9.0/bugfix/CLDSRV-620/strip-trailing-checksums) for [commit](https://github.com/scality/cloudserver/commit/0fe6084edf3fe8c5eb3d6ded23c3621ff5a4becb) did not succeed in branch [w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums](https://github.com/scality/cloudserver/compare/development/9.0...w/9.0/bugfix/CLDSRV-620/strip-trailing-checksums)



*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR
user: login: bert-e
created_at: 2025-03-17T20:22:35Z
body: # 


I have successfully merged the changeset of this pull request
into targetted development branches:


* :heavy_check_mark: `development/7.10`

* :heavy_check_mark: `development/7.70`

* :heavy_check_mark: `development/8.8`

* :heavy_check_mark: `development/9.0`



The following branches have **NOT** changed:

* `development/7.4`



Please check the status of the associated issue CLDSRV-620.

Goodbye fredmnl.




*The following options are set:* **approve, create_integration_branches**


author_association: CONTRIBUTOR