Repository: nodejs/node
Pull request: inspector: support undici traffic data inspection

Support undici sent and received data inspection in Chrome DevTools.

<!--
Before submitting a pull request, please read:

- the CONTRIBUTING guide at https://github.com/nodejs/node/blob/HEAD/CONTRIBUTING.md
- the commit message formatting guidelines at
  https://github.com/nodejs/node/blob/HEAD/doc/contributing/pull-requests.md#commit-message-guidelines

For code changes:
1. Include tests for any bug fixes or new features.
2. Update documentation if relevant.
3. Ensure that `make -j4 test` (UNIX), or `vcbuild test` (Windows) passes.

If you believe this PR should be highlighted in the Node.js CHANGELOG
please add the `notable-change` label.

Developer's Certificate of Origin 1.1

By making a contribution to this project, I certify that:

(a) The contribution was created in whole or in part by me and I
    have the right to submit it under the open source license
    indicated in the file; or

(b) The contribution is based upon previous work that, to the best
    of my knowledge, is covered under an appropriate open source
    license and I have the right under that license to submit that
    work with modifications, whether created in whole or in part
    by me, under the same open source license (unless I am
    permitted to submit under a different license), as indicated
    in the file; or

(c) The contribution was provided directly to me by some other
    person who certified (a), (b) or (c) and I have not modified
    it.

(d) I understand and agree that this project and the contribution
    are public and that a record of the contribution (including all
    personal information I submit with it, including my sign-off) is
    maintained indefinitely and may be redistributed consistent with
    this project or the open source license(s) involved.
-->


Changed files:
filename: lib/internal/inspector/network.js
patch: @@ -2,6 +2,7 @@
 
 const {
   NumberMAX_SAFE_INTEGER,
+  StringPrototypeToLowerCase,
   Symbol,
 } = primordials;
 
@@ -52,8 +53,8 @@ function sniffMimeType(contentType) {
   let charset;
   try {
     const mimeTypeObj = new MIMEType(contentType);
-    mimeType = mimeTypeObj.essence || '';
-    charset = mimeTypeObj.params.get('charset') || '';
+    mimeType = StringPrototypeToLowerCase(mimeTypeObj.essence || '');
+    charset = StringPrototypeToLowerCase(mimeTypeObj.params.get('charset') || '');
   } catch {
     mimeType = '';
     charset = '';

filename: lib/internal/inspector/network_undici.js
patch: @@ -2,6 +2,7 @@
 
 const {
   DateNow,
+  StringPrototypeToLowerCase,
 } = primordials;
 
 const {
@@ -13,16 +14,25 @@ const {
 } = require('internal/inspector/network');
 const dc = require('diagnostics_channel');
 const { Network } = require('inspector');
+const { Buffer } = require('buffer');
 
 // Convert an undici request headers array to a plain object (Map<string, string>)
 function requestHeadersArrayToDictionary(headers) {
   const dict = {};
+  let charset;
+  let mimeType;
   for (let idx = 0; idx < headers.length; idx += 2) {
     const key = `${headers[idx]}`;
     const value = `${headers[idx + 1]}`;
     dict[key] = value;
+
+    if (StringPrototypeToLowerCase(key) === 'content-type') {
+      const result = sniffMimeType(value);
+      charset = result.charset;
+      mimeType = result.mimeType;
+    }
   }
-  return dict;
+  return [dict, charset, mimeType];
 };
 
 // Convert an undici response headers array to a plain object (Map<string, string>)
@@ -32,7 +42,7 @@ function responseHeadersArrayToDictionary(headers) {
   let mimeType;
   for (let idx = 0; idx < headers.length; idx += 2) {
     const key = `${headers[idx]}`;
-    const lowerCasedKey = key.toLowerCase();
+    const lowerCasedKey = StringPrototypeToLowerCase(key);
     const value = `${headers[idx + 1]}`;
     const prevValue = dict[key];
 
@@ -63,8 +73,7 @@ function onClientRequestStart({ request }) {
   const url = `${request.origin}${request.path}`;
   request[kInspectorRequestId] = getNextRequestId();
 
-  const headers = requestHeadersArrayToDictionary(request.headers);
-  const { charset } = sniffMimeType(headers);
+  const { 0: headers, 1: charset } = requestHeadersArrayToDictionary(request.headers);
 
   Network.requestWillBeSent({
     requestId: request[kInspectorRequestId],
@@ -74,7 +83,8 @@ function onClientRequestStart({ request }) {
     request: {
       url,
       method: request.method,
-      headers: requestHeadersArrayToDictionary(request.headers),
+      headers: headers,
+      hasPostData: request.body != null,
     },
   });
 }
@@ -97,6 +107,40 @@ function onClientRequestError({ request, error }) {
   });
 }
 
+/**
+ * When a chunk of the request body is being sent, cache it until `getRequestPostData` request.
+ * https://chromedevtools.github.io/devtools-protocol/1-3/Network/#method-getRequestPostData
+ * @param {{ request: undici.Request, chunk: Uint8Array | string }} event
+ */
+function onClientRequestBodyChunkSent({ request, chunk }) {
+  if (typeof request[kInspectorRequestId] !== 'string') {
+    return;
+  }
+
+  const buffer = Buffer.from(chunk);
+  Network.dataSent({
+    requestId: request[kInspectorRequestId],
+    timestamp: getMonotonicTime(),
+    dataLength: buffer.byteLength,
+    data: buffer,
+  });
+}
+
+/**
+ * Mark a request body as fully sent.
+ * @param {{request: undici.Request}} event
+ */
+function onClientRequestBodySent({ request }) {
+  if (typeof request[kInspectorRequestId] !== 'string') {
+    return;
+  }
+
+  Network.dataSent({
+    requestId: request[kInspectorRequestId],
+    finished: true,
+  });
+}
+
 /**
  * When response headers are received, emit Network.responseReceived event.
  * https://chromedevtools.github.io/devtools-protocol/1-3/Network/#event-responseReceived
@@ -126,6 +170,27 @@ function onClientResponseHeaders({ request, response }) {
   });
 }
 
+/**
+ * When a chunk of the response body has been received, cache it until `getResponseBody` request
+ * https://chromedevtools.github.io/devtools-protocol/1-3/Network/#method-getResponseBody or
+ * stream it with `streamResourceContent` request.
+ * https://chromedevtools.github.io/devtools-protocol/tot/Network/#method-streamResourceContent
+ * @param {{ request: undici.Request, chunk: Uint8Array | string }} event
+ */
+function onClientRequestBodyChunkReceived({ request, chunk }) {
+  if (typeof request[kInspectorRequestId] !== 'string') {
+    return;
+  }
+
+  Network.dataReceived({
+    requestId: request[kInspectorRequestId],
+    timestamp: getMonotonicTime(),
+    dataLength: chunk.byteLength,
+    encodedDataLength: chunk.byteLength,
+    data: chunk,
+  });
+}
+
 /**
  * When a response is completed, emit Network.loadingFinished event.
  * https://chromedevtools.github.io/devtools-protocol/1-3/Network/#event-loadingFinished
@@ -146,13 +211,19 @@ function enable() {
   dc.subscribe('undici:request:error', onClientRequestError);
   dc.subscribe('undici:request:headers', onClientResponseHeaders);
   dc.subscribe('undici:request:trailers', onClientResponseFinish);
+  dc.subscribe('undici:request:bodyChunkSent', onClientRequestBodyChunkSent);
+  dc.subscribe('undici:request:bodySent', onClientRequestBodySent);
+  dc.subscribe('undici:request:bodyChunkReceived', onClientRequestBodyChunkReceived);
 }
 
 function disable() {
   dc.unsubscribe('undici:request:create', onClientRequestStart);
   dc.unsubscribe('undici:request:error', onClientRequestError);
   dc.unsubscribe('undici:request:headers', onClientResponseHeaders);
   dc.unsubscribe('undici:request:trailers', onClientResponseFinish);
+  dc.unsubscribe('undici:request:bodyChunkSent', onClientRequestBodyChunkSent);
+  dc.unsubscribe('undici:request:bodySent', onClientRequestBodySent);
+  dc.unsubscribe('undici:request:bodyChunkReceived', onClientRequestBodyChunkReceived);
 }
 
 module.exports = {

filename: test/parallel/test-inspector-network-fetch.js
patch: @@ -5,6 +5,7 @@ const common = require('../common');
 common.skipIfInspectorDisabled();
 
 const assert = require('node:assert');
+const { once } = require('node:events');
 const { addresses } = require('../common/internet');
 const fixtures = require('../common/fixtures');
 const http = require('node:http');
@@ -42,11 +43,19 @@ const setResponseHeaders = (res) => {
 const handleRequest = (req, res) => {
   const path = req.url;
   switch (path) {
-    case '/hello-world':
+    case '/hello-world': {
       setResponseHeaders(res);
-      res.writeHead(200);
-      res.end('hello world\n');
+      const chunks = [];
+      req.on('data', (chunk) => {
+        chunks.push(chunk);
+      });
+      req.on('end', () => {
+        assert.strictEqual(Buffer.concat(chunks).toString(), 'foobar');
+        res.writeHead(200);
+        res.end('hello world\n');
+      });
       break;
+    }
     default:
       assert(false, `Unexpected path: ${path}`);
   }
@@ -73,127 +82,126 @@ function findFrameInInitiator(scriptName, initiator) {
   return frame;
 }
 
-const testHttpGet = () => new Promise((resolve, reject) => {
-  session.on('Network.requestWillBeSent', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(params.request.url, `http://127.0.0.1:${httpServer.address().port}/hello-world`);
-    assert.strictEqual(params.request.method, 'GET');
-    assert.strictEqual(typeof params.request.headers, 'object');
-    assert.strictEqual(params.request.headers['accept-language'], 'en-US');
-    assert.strictEqual(params.request.headers.cookie, 'k1=v1; k2=v2');
-    assert.strictEqual(params.request.headers.age, '1000');
-    assert.strictEqual(params.request.headers['x-header1'], 'value1, value2');
-    assert.strictEqual(typeof params.timestamp, 'number');
-    assert.strictEqual(typeof params.wallTime, 'number');
-
-    assert.strictEqual(typeof params.initiator, 'object');
-    assert.strictEqual(params.initiator.type, 'script');
-    assert.ok(findFrameInInitiator(__filename, params.initiator));
-  }));
-  session.on('Network.responseReceived', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(typeof params.timestamp, 'number');
-    assert.strictEqual(params.type, 'Fetch');
-    assert.strictEqual(params.response.status, 200);
-    assert.strictEqual(params.response.statusText, 'OK');
-    assert.strictEqual(params.response.url, `http://127.0.0.1:${httpServer.address().port}/hello-world`);
-    assert.strictEqual(typeof params.response.headers, 'object');
-    assert.strictEqual(params.response.headers.server, 'node');
-    assert.strictEqual(params.response.headers.etag, '12345');
-    assert.strictEqual(params.response.headers['Set-Cookie'], 'key1=value1\nkey2=value2');
-    assert.strictEqual(params.response.headers['x-header2'], 'value1, value2');
-    assert.strictEqual(params.response.mimeType, 'text/plain');
-    assert.strictEqual(params.response.charset, 'utf-8');
-  }));
-  session.on('Network.loadingFinished', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(typeof params.timestamp, 'number');
-    resolve();
-  }));
-
-  fetch(`http://127.0.0.1:${httpServer.address().port}/hello-world`, {
-    headers: requestHeaders,
-  }).then(common.mustCall());
-});
+function verifyRequestWillBeSent({ method, params }, expect) {
+  assert.strictEqual(method, 'Network.requestWillBeSent');
+
+  assert.ok(params.requestId.startsWith('node-network-event-'));
+  assert.strictEqual(params.request.url, expect.url);
+  assert.strictEqual(params.request.method, expect.method);
+  assert.strictEqual(typeof params.request.headers, 'object');
+  assert.strictEqual(params.request.headers['accept-language'], 'en-US');
+  assert.strictEqual(params.request.headers.cookie, 'k1=v1; k2=v2');
+  assert.strictEqual(params.request.headers.age, '1000');
+  assert.strictEqual(params.request.headers['x-header1'], 'value1, value2');
+  assert.strictEqual(typeof params.timestamp, 'number');
+  assert.strictEqual(typeof params.wallTime, 'number');
+
+  assert.strictEqual(typeof params.initiator, 'object');
+  assert.strictEqual(params.initiator.type, 'script');
+  assert.ok(findFrameInInitiator(__filename, params.initiator));
+
+  return params;
+}
+
+function verifyResponseReceived({ method, params }, expect) {
+  assert.strictEqual(method, 'Network.responseReceived');
+
+  assert.ok(params.requestId.startsWith('node-network-event-'));
+  assert.strictEqual(typeof params.timestamp, 'number');
+  assert.strictEqual(params.type, 'Fetch');
+  assert.strictEqual(params.response.status, 200);
+  assert.strictEqual(params.response.statusText, 'OK');
+  assert.strictEqual(params.response.url, expect.url);
+  assert.strictEqual(typeof params.response.headers, 'object');
+  assert.strictEqual(params.response.headers.server, 'node');
+  assert.strictEqual(params.response.headers.etag, '12345');
+  assert.strictEqual(params.response.headers['Set-Cookie'], 'key1=value1\nkey2=value2');
+  assert.strictEqual(params.response.headers['x-header2'], 'value1, value2');
+  assert.strictEqual(params.response.mimeType, 'text/plain');
+  assert.strictEqual(params.response.charset, 'utf-8');
+
+  return params;
+}
+
+function verifyLoadingFinished({ method, params }) {
+  assert.strictEqual(method, 'Network.loadingFinished');
+
+  assert.ok(params.requestId.startsWith('node-network-event-'));
+  assert.strictEqual(typeof params.timestamp, 'number');
+  return params;
+}
+
+function verifyLoadingFailed({ method, params }) {
+  assert.strictEqual(method, 'Network.loadingFailed');
+
+  assert.ok(params.requestId.startsWith('node-network-event-'));
+  assert.strictEqual(typeof params.timestamp, 'number');
+  assert.strictEqual(params.type, 'Fetch');
+  assert.strictEqual(typeof params.errorText, 'string');
+}
+
+async function testRequest(url) {
+  const requestWillBeSentFuture = once(session, 'Network.requestWillBeSent')
+    .then(([event]) => verifyRequestWillBeSent(event, { url, method: 'POST' }));
+
+  const responseReceivedFuture = once(session, 'Network.responseReceived')
+    .then(([event]) => verifyResponseReceived(event, { url }));
 
-const testHttpsGet = () => new Promise((resolve, reject) => {
-  session.on('Network.requestWillBeSent', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(params.request.url, `https://127.0.0.1:${httpsServer.address().port}/hello-world`);
-    assert.strictEqual(params.request.method, 'GET');
-    assert.strictEqual(typeof params.request.headers, 'object');
-    assert.strictEqual(params.request.headers['accept-language'], 'en-US');
-    assert.strictEqual(params.request.headers.cookie, 'k1=v1; k2=v2');
-    assert.strictEqual(params.request.headers.age, '1000');
-    assert.strictEqual(params.request.headers['x-header1'], 'value1, value2');
-    assert.strictEqual(typeof params.timestamp, 'number');
-    assert.strictEqual(typeof params.wallTime, 'number');
-  }));
-  session.on('Network.responseReceived', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(typeof params.timestamp, 'number');
-    assert.strictEqual(params.type, 'Fetch');
-    assert.strictEqual(params.response.status, 200);
-    assert.strictEqual(params.response.statusText, 'OK');
-    assert.strictEqual(params.response.url, `https://127.0.0.1:${httpsServer.address().port}/hello-world`);
-    assert.strictEqual(typeof params.response.headers, 'object');
-    assert.strictEqual(params.response.headers.server, 'node');
-    assert.strictEqual(params.response.headers.etag, '12345');
-    assert.strictEqual(params.response.headers['Set-Cookie'], 'key1=value1\nkey2=value2');
-    assert.strictEqual(params.response.headers['x-header2'], 'value1, value2');
-    assert.strictEqual(params.response.mimeType, 'text/plain');
-    assert.strictEqual(params.response.charset, 'utf-8');
-  }));
-  session.on('Network.loadingFinished', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(typeof params.timestamp, 'number');
-    resolve();
-  }));
-
-  fetch(`https://127.0.0.1:${httpsServer.address().port}/hello-world`, {
+  const loadingFinishedFuture = once(session, 'Network.loadingFinished')
+    .then(([event]) => verifyLoadingFinished(event));
+
+  await fetch(url, {
+    method: 'POST',
+    body: 'foobar',
     headers: requestHeaders,
-  }).then(common.mustCall());
-});
+  });
 
-const testHttpError = () => new Promise((resolve, reject) => {
-  session.on('Network.requestWillBeSent', common.mustCall());
-  session.on('Network.loadingFailed', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(typeof params.timestamp, 'number');
-    assert.strictEqual(params.type, 'Fetch');
-    assert.strictEqual(typeof params.errorText, 'string');
-    resolve();
-  }));
-  session.on('Network.responseReceived', common.mustNotCall());
-  session.on('Network.loadingFinished', common.mustNotCall());
+  await requestWillBeSentFuture;
+  const responseReceived = await responseReceivedFuture;
+  const loadingFinished = await loadingFinishedFuture;
+  assert.ok(loadingFinished.timestamp >= responseReceived.timestamp);
 
-  fetch(`http://${addresses.INVALID_HOST}`).catch(common.mustCall());
-});
+  const requestBody = await session.post('Network.getRequestPostData', {
+    requestId: responseReceived.requestId,
+  });
+  assert.strictEqual(requestBody.postData, 'foobar');
 
+  const responseBody = await session.post('Network.getResponseBody', {
+    requestId: responseReceived.requestId,
+  });
+  assert.strictEqual(responseBody.base64Encoded, false);
+  assert.strictEqual(responseBody.body, 'hello world\n');
+}
 
-const testHttpsError = () => new Promise((resolve, reject) => {
-  session.on('Network.requestWillBeSent', common.mustCall());
-  session.on('Network.loadingFailed', common.mustCall(({ params }) => {
-    assert.ok(params.requestId.startsWith('node-network-event-'));
-    assert.strictEqual(typeof params.timestamp, 'number');
-    assert.strictEqual(params.type, 'Fetch');
-    assert.strictEqual(typeof params.errorText, 'string');
-    resolve();
-  }));
+async function testRequestError(url) {
+  const requestWillBeSentFuture = once(session, 'Network.requestWillBeSent')
+    .then(([event]) => verifyRequestWillBeSent(event, { url, method: 'GET' }));
   session.on('Network.responseReceived', common.mustNotCall());
   session.on('Network.loadingFinished', common.mustNotCall());
 
-  fetch(`https://${addresses.INVALID_HOST}`).catch(common.mustCall());
-});
+  const loadingFailedFuture = once(session, 'Network.loadingFailed')
+    .then(([event]) => verifyLoadingFailed(event));
+
+  fetch(url, {
+    headers: requestHeaders,
+  }).catch(common.mustCall());
+
+  await requestWillBeSentFuture;
+  await loadingFailedFuture;
+}
 
 const testNetworkInspection = async () => {
-  await testHttpGet();
+  // HTTP
+  await testRequest(`http://127.0.0.1:${httpServer.address().port}/hello-world`);
   session.removeAllListeners();
-  await testHttpsGet();
+  // HTTPS
+  await testRequest(`https://127.0.0.1:${httpsServer.address().port}/hello-world`);
   session.removeAllListeners();
-  await testHttpError();
+  // HTTP with invalid host
+  await testRequestError(`http://${addresses.INVALID_HOST}/`);
   session.removeAllListeners();
-  await testHttpsError();
+  // HTTPS with invalid host
+  await testRequestError(`https://${addresses.INVALID_HOST}/`);
   session.removeAllListeners();
 };