Repository: CheckerNetwork/spark-checker
Pull request #124: feat: add client-side max request duration timeout to Spark (#99)
Author: login: Hany-Almnaem
Created: 2025-03-24T15:02:01Z | Merged: 2025-03-27T09:05:26Z

This PR adds a client-side max request duration timeout to prevent retrievals from lingering beyond a reasonable threshold, resolving issue #99.

- Adds `maxTimeout` tracking in `fetchCAR`
- Sets hard timeout (e.g. 90s) in addition to progress timeout (60s)
- Adds a test to ensure max timeout aborts stalled connections



Diff:
diff --git a/lib/constants.js b/lib/constants.js
index 9bd7e58..e19d9e4 100644
--- a/lib/constants.js
+++ b/lib/constants.js
@@ -1,6 +1,8 @@
-export const SPARK_VERSION = '1.18.1'
+export const SPARK_VERSION = '1.18.2'
 export const MAX_CAR_SIZE = 200 * 1024 * 1024 // 200 MB
 export const APPROX_ROUND_LENGTH_IN_MS = 20 * 60_000 // 20 minutes
+export const MAX_JITTER_BETWEEN_TASKS_IN_MS = 10_000 // 10 seconds
 export const RPC_URL = 'https://api.node.glif.io/'
 export const RPC_AUTH = 'KZLIUb9ejreYOm-mZFM3UNADE0ux6CrHjxnS2D2Qgb8='
 export const MINER_TO_PEERID_CONTRACT_ADDRESS = '0x14183aD016Ddc83D638425D6328009aa390339Ce' // Contract address on the Filecoin EVM
+export const MAX_REQUEST_DURATION_MS = 90_000
\ No newline at end of file
diff --git a/lib/spark.js b/lib/spark.js
index 3f1aca4..190241d 100644
--- a/lib/spark.js
+++ b/lib/spark.js
@@ -1,7 +1,7 @@
 /* global Zinnia */
 
 import { ActivityState } from './activity-state.js'
-import { SPARK_VERSION, MAX_CAR_SIZE, APPROX_ROUND_LENGTH_IN_MS } from './constants.js'
+import { SPARK_VERSION, MAX_CAR_SIZE, APPROX_ROUND_LENGTH_IN_MS, MAX_JITTER_BETWEEN_TASKS_IN_MS, MAX_REQUEST_DURATION_MS } from './constants.js'
 import { queryTheIndex } from './ipni-client.js'
 import { assertOkResponse } from './http-assertions.js'
 import { getIndexProviderPeerId as defaultGetIndexProvider } from './miner-info.js'
@@ -83,20 +83,27 @@ export default class Spark {
     }
   }
 
-  async fetchCAR (protocol, address, cid, stats) {
+  async fetchCAR (protocol, address, cid, stats, { maxRequestDurationMs = MAX_REQUEST_DURATION_MS } = {}) {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
-    let timeout
+
+    let requestIdleTimeout
+    let maxDurationTimeout
+
     const resetTimeout = () => {
-      if (timeout) {
-        clearTimeout(timeout)
+      if (requestIdleTimeout) {
+        clearTimeout(requestIdleTimeout)
       }
-      timeout = setTimeout(() => {
+      requestIdleTimeout = setTimeout(() => {
         stats.timeout = true
         controller.abort()
       }, 60_000)
     }
+    maxDurationTimeout = setTimeout(() => {
+      stats.timeout = true
+      controller.abort()
+    }, maxRequestDurationMs)
 
     // WebCrypto API does not support streams yet, the hashing function requires entire data
     // to be provided at once. See https://github.com/w3c/webcrypto/issues/73
@@ -157,7 +164,8 @@ export default class Spark {
         stats.statusCode = mapErrorToStatusCode(err)
       }
     } finally {
-      clearTimeout(timeout)
+      clearTimeout(requestIdleTimeout)
+      clearTimeout(maxDurationTimeout)
     }
 
     stats.endAt = new Date()
@@ -235,6 +243,7 @@ export default class Spark {
       const duration = Date.now() - started
       const delay = calculateDelayBeforeNextTask({
         roundLengthInMs: APPROX_ROUND_LENGTH_IN_MS,
+        maxJitterInMs: MAX_JITTER_BETWEEN_TASKS_IN_MS,
         maxTasksPerRound: this.#tasker.maxTasksPerRound,
         lastTaskDurationInMs: duration
       })
@@ -263,17 +272,24 @@ export default class Spark {
 /**
  * @param {object} args
  * @param {number} args.roundLengthInMs
+ * @param {number} [args.maxJitterInMs=0]
  * @param {number} args.maxTasksPerRound
  * @param {number} args.lastTaskDurationInMs
  */
 export function calculateDelayBeforeNextTask ({
   roundLengthInMs,
+  maxJitterInMs = 0,
   maxTasksPerRound,
   lastTaskDurationInMs
 }) {
   const baseDelay = roundLengthInMs / maxTasksPerRound
   const delay = baseDelay - lastTaskDurationInMs
-  return Math.min(delay, 60_000)
+  const base = Math.min(delay, 60_000)
+
+  // Introduce some jitter to avoid all clients querying cid.contact at the same time
+  const jitter = Math.round(Math.random() * maxJitterInMs)
+
+  return base + jitter
 }
 
 export function newStats () {
diff --git a/test/spark.js b/test/spark.js
index a240e89..ae9ab8b 100644
--- a/test/spark.js
+++ b/test/spark.js
@@ -2,7 +2,7 @@
 
 import Spark, { calculateDelayBeforeNextTask, newStats } from '../lib/spark.js'
 import { test } from 'zinnia:test'
-import { assertInstanceOf, assertEquals, assertArrayIncludes } from 'zinnia:assert'
+import { assertInstanceOf, assertEquals, assertArrayIncludes, assertNotEquals, assertLessOrEqual, assertGreaterOrEqual } from 'zinnia:assert'
 import { SPARK_VERSION } from '../lib/constants.js'
 
 const KNOWN_CID = 'bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq'
@@ -363,3 +363,88 @@ test('calculateDelayBeforeNextTask() handles one task per round', () => {
   })
   assertEquals(delay, 60_000)
 })
+
+test('calculateDelayBeforeNextTask() introduces random jitter', () => {
+  const getDelay = () => calculateDelayBeforeNextTask({
+    lastTaskDurationInMs: 3_000,
+
+    // one task every 10 seconds (on average)
+    roundLengthInMs: 60_000,
+    maxTasksPerRound: 6,
+
+    // jitter up to 1 second
+    maxJitterInMs: 1_000
+  })
+
+  const delay1 = getDelay()
+  const delay2 = getDelay()
+
+  assertGreaterOrEqual(delay1, 7_000)
+  assertLessOrEqual(delay1, 7_000 + 1_000)
+
+  assertNotEquals(
+    delay1,
+    delay2,
+    `Expected delay values to be different because of jitter. Actual value: ${delay1}`
+  )
+  assertLessOrEqual(
+    Math.abs(delay1 - delay2),
+    1_000,
+    `expected delay values to be within 1 second of each other. Actual values: ${delay1} <> ${delay2}`
+  )
+})
+
+test('calculateDelayBeforeNextTask() introduces random jitter for zero tasks in round', () => {
+  const getDelay = () => calculateDelayBeforeNextTask({
+    maxTasksPerRound: 0,
+
+    // jitter up to 1 second
+    maxJitterInMs: 1_000,
+
+    // the values below are not important
+    roundLengthInMs: 12345,
+    lastTaskDurationInMs: 12
+  })
+
+  const delay1 = getDelay()
+  const delay2 = getDelay()
+
+  assertNotEquals(
+    delay1,
+    delay2,
+    `Expected delay values to be different because of jitter. Actual value: ${delay1}`
+  )
+  assertLessOrEqual(
+    Math.abs(delay1 - delay2),
+    1_000,
+    `expected delay values to be within 1 second of each other. Actual values: ${delay1} <> ${delay2}`
+  )
+})
+
+test('fetchCAR triggers timeout after long retrieval', async () => {
+  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+  const fetch = async (_url, { signal }) => {
+    return {
+      status: 200,
+      ok: true,
+      body: (async function * () {
+        while (true) {
+          if (signal.aborted) {
+            throw new DOMException('Aborted', 'AbortError')
+          }
+          yield new Uint8Array([0])
+          await sleep(500)
+        }
+      })()
+    }
+  }
+
+  const spark = new Spark({ fetch })
+  const stats = newStats()
+
+  await spark.fetchCAR('http', '/dns/example.com/tcp/80/http', KNOWN_CID, stats, {
+    maxRequestDurationMs: 0
+  })
+
+  assertEquals(stats.timeout, true)
+})


Reviews:
user: login: juliangruber
body: Great start!
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-03-24T15:10:23Z
user: login: bajtos
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-24T15:43:48Z
user: login: juliangruber
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-03-25T07:01:06Z
user: login: Hany-Almnaem
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T17:17:08Z
user: login: juliangruber
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-25T22:27:31Z
user: login: Hany-Almnaem
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-26T11:07:38Z
user: login: Hany-Almnaem
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-26T11:09:42Z
user: login: bajtos
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-26T12:01:50Z
user: login: Hany-Almnaem
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-26T12:20:39Z
user: login: juliangruber
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-03-26T12:48:13Z
user: login: juliangruber
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-27T07:26:39Z

Review comments:
diff_hunk: @@ -87,6 +87,9 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+    const MAX_REQUEST_DURATION_MS = 90_000 
path: lib/spark.js
user: login: juliangruber
body: This definition should go in `lib/constants.js`
created_at: 2025-03-24T15:08:59Z
author_association: MEMBER
diff_hunk: @@ -97,6 +100,11 @@ export default class Spark {
         controller.abort()
       }, 60_000)
     }
+    // Start max duration timeout
+maxDurationTimeout = setTimeout(() => {
+  stats.maxTimeout = true
+  controller.abort()
+}, MAX_REQUEST_DURATION_MS)
path: lib/spark.js
user: login: juliangruber
body: indentation is off
created_at: 2025-03-24T15:09:20Z
author_association: MEMBER
diff_hunk: @@ -279,6 +289,7 @@ export function calculateDelayBeforeNextTask ({
 export function newStats () {
   return {
     timeout: false,
+    maxTimeout: false,
path: lib/spark.js
user: login: juliangruber
body: @bajtos do you think we should add a new field, or rather overload the existing `timeout` field for this?
created_at: 2025-03-24T15:09:56Z
author_association: MEMBER
diff_hunk: @@ -158,6 +166,8 @@ export default class Spark {
       }
     } finally {
       clearTimeout(timeout)
+      // clear max duration timeout
path: lib/spark.js
user: login: juliangruber
body: ```suggestion
```

too verbose
created_at: 2025-03-24T15:10:05Z
author_association: MEMBER
diff_hunk: @@ -97,6 +100,11 @@ export default class Spark {
         controller.abort()
       }, 60_000)
     }
+    // Start max duration timeout
path: lib/spark.js
user: login: juliangruber
body: ```suggestion
```
created_at: 2025-03-24T15:10:12Z
author_association: MEMBER
diff_hunk: @@ -279,6 +289,7 @@ export function calculateDelayBeforeNextTask ({
 export function newStats () {
   return {
     timeout: false,
+    maxTimeout: false,
path: lib/spark.js
user: login: bajtos
body: I prefer to use the existing field `timeout` until we have a need to discriminate between different timeout types.
created_at: 2025-03-24T15:43:48Z
author_association: MEMBER
diff_hunk: @@ -363,3 +363,47 @@ test('calculateDelayBeforeNextTask() handles one task per round', () => {
   })
   assertEquals(delay, 60_000)
 })
+test('fetchCAR triggers timeout after long retrieval', async () => {
+  const MAX_TIMEOUT_MS = 1000 // 1 second max duration for testing
+  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+  // Simulate a fetch() that keeps sending data slowly forever
+  const fetch = async (_url, { signal }) => {
+    return {
+      status: 200,
+      ok: true,
+      body: (async function * () {
+        // Simulate long, slow stream
+        let i = 0
path: test/spark.js
user: login: juliangruber
body: ```suggestion
```

We can just always respond with 0
created_at: 2025-03-25T06:54:41Z
author_association: MEMBER
diff_hunk: @@ -363,3 +363,47 @@ test('calculateDelayBeforeNextTask() handles one task per round', () => {
   })
   assertEquals(delay, 60_000)
 })
+test('fetchCAR triggers timeout after long retrieval', async () => {
+  const MAX_TIMEOUT_MS = 1000 // 1 second max duration for testing
+  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+  // Simulate a fetch() that keeps sending data slowly forever
+  const fetch = async (_url, { signal }) => {
+    return {
+      status: 200,
+      ok: true,
+      body: (async function * () {
+        // Simulate long, slow stream
+        let i = 0
+        while (true) {
+          if (signal.aborted) {
+            throw new DOMException('Aborted', 'AbortError')
+          }
+          yield new Uint8Array([i++])
path: test/spark.js
user: login: juliangruber
body: ```suggestion
          yield new Uint8Array([0])
```
created_at: 2025-03-25T06:54:50Z
author_association: MEMBER
diff_hunk: @@ -87,6 +87,8 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+
+    let maxDurationTimeout
     let timeout
path: lib/spark.js
user: login: juliangruber
body: To me it's not immediately clear what the difference is between `maxDurationTimeout` and `timeout`. I propose we find a better name for `timeout`. What do you think about `requestIdleTimeout`?
created_at: 2025-03-25T06:56:39Z
author_association: MEMBER
diff_hunk: @@ -363,3 +363,47 @@ test('calculateDelayBeforeNextTask() handles one task per round', () => {
   })
   assertEquals(delay, 60_000)
 })
+test('fetchCAR triggers timeout after long retrieval', async () => {
+  const MAX_TIMEOUT_MS = 1000 // 1 second max duration for testing
+  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+  // Simulate a fetch() that keeps sending data slowly forever
+  const fetch = async (_url, { signal }) => {
+    return {
+      status: 200,
+      ok: true,
+      body: (async function * () {
+        // Simulate long, slow stream
+        let i = 0
+        while (true) {
+          if (signal.aborted) {
+            throw new DOMException('Aborted', 'AbortError')
+          }
+          yield new Uint8Array([i++])
+          await sleep(500) // emit a chunk every 500ms
+        }
+      })()
+    }
+  }
+
+  const spark = new Spark({ fetch })
+  const stats = newStats()
+
+  // Monkey-patch the timeout duration to 1 second for test
+  const original = globalThis.setTimeout
path: test/spark.js
user: login: juliangruber
body: I was trying to look at how we test the current `timeout` implementation, to take inspiration, but was surprised to see we don't have a test for that yet.

I think we have two ways:
1. Monkey-path `setTimeout` as you did here. If we do that, we need to wrap all following code in a `try/finally`, so that we can restore the original implementation in the `finally` block, and not produce unintended side effects, should the code fail.
2. Add an argument for `MAX_REQUEST_DURATION_MS` to `fetchCAR`, which defaults to the constant:
```js
async fetchCAR (protocol, address, cid, stats, { maxFetchDurationMs = MAX_FETCH_DURATION_MS } = {}) {
```
This way we don't need to patch anything, and can for the test simply pass `maxFetchDurationMs: 0`. This way the timeout should always trigger.

I prefer the second solution.
created_at: 2025-03-25T07:01:00Z
author_association: MEMBER
diff_hunk: @@ -87,6 +87,8 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+
+    let maxDurationTimeout
     let timeout
path: lib/spark.js
user: login: Hany-Almnaem
body: > What do you think about `requestIdleTimeout`

That's a very thoughtful suggestion. Just to clarify — If we name the variable requestIdleTimeout and set stats.requestIdleTimeout. For the overall timeout (maxDurationTimeout), can we setting stats.timeout.

 For example:

Use requestIdleTimeout → stats.requestIdleTimeout

Use maxDurationTimeout → stats.timeout

Or you have another idea?
created_at: 2025-03-25T17:17:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -87,6 +87,8 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+
+    let maxDurationTimeout
     let timeout
path: lib/spark.js
user: login: juliangruber
body: Let's keep `stats.timeout` for both timeouts, we don't need the distinction yet. But the timeout id variable name sounds good as `requestIdleTimeout`
created_at: 2025-03-25T22:27:31Z
author_association: MEMBER
diff_hunk: @@ -87,6 +87,8 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+
+    let maxDurationTimeout
     let timeout
path: lib/spark.js
user: login: Hany-Almnaem
body: > Let's keep `stats.timeout` for both timeouts

I’ve applied the suggested renaming:

1. requestIdleTimeout now handles idle progress
2. maxDurationTimeout controls the overall duration
3. Both set stats.timeout = true as discussed
created_at: 2025-03-26T11:07:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -363,3 +363,47 @@ test('calculateDelayBeforeNextTask() handles one task per round', () => {
   })
   assertEquals(delay, 60_000)
 })
+test('fetchCAR triggers timeout after long retrieval', async () => {
+  const MAX_TIMEOUT_MS = 1000 // 1 second max duration for testing
+  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+
+  // Simulate a fetch() that keeps sending data slowly forever
+  const fetch = async (_url, { signal }) => {
+    return {
+      status: 200,
+      ok: true,
+      body: (async function * () {
+        // Simulate long, slow stream
+        let i = 0
+        while (true) {
+          if (signal.aborted) {
+            throw new DOMException('Aborted', 'AbortError')
+          }
+          yield new Uint8Array([i++])
+          await sleep(500) // emit a chunk every 500ms
+        }
+      })()
+    }
+  }
+
+  const spark = new Spark({ fetch })
+  const stats = newStats()
+
+  // Monkey-patch the timeout duration to 1 second for test
+  const original = globalThis.setTimeout
path: test/spark.js
user: login: Hany-Almnaem
body: > This way we don't need to patch anything

I’ve updated the test to avoid monkey-patching and reflect the final logic. Let me know if there are any further adjustments.
created_at: 2025-03-26T11:09:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -87,6 +87,8 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+
+    let maxDurationTimeout
     let timeout
path: lib/spark.js
user: login: bajtos
body: +1 to keep `stats.timeout`. This name is used in spark-api and spark-evaluate, too. We would have to update those two places if we changed the name here.
created_at: 2025-03-26T12:01:50Z
author_association: MEMBER
diff_hunk: @@ -87,6 +87,8 @@ export default class Spark {
     // Abort if no progress was made for 60 seconds
     const controller = new AbortController()
     const { signal } = controller
+
+    let maxDurationTimeout
     let timeout
path: lib/spark.js
user: login: Hany-Almnaem
body: > +1 to keep `stats.timeout`.

Thanks! Already done in the latest commit. Here's the updated part:

requestIdleTimeout = setTimeout(() => {
  stats.timeout = true
  controller.abort()
}, 60_000)

maxDurationTimeout = setTimeout(() => {
  stats.timeout = true
  controller.abort()
}, maxFetchDurationMs)

created_at: 2025-03-26T12:20:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -83,20 +83,27 @@ export default class Spark {
     }
   }
 
-  async fetchCAR (protocol, address, cid, stats) {
+  async fetchCAR (protocol, address, cid, stats, { maxFetchDurationMs = MAX_REQUEST_DURATION_MS } = {}) {
path: lib/spark.js
user: login: juliangruber
body: inconsistent naming, should be either max fetch duration or max request duration everywhere
created_at: 2025-03-26T12:47:21Z
author_association: MEMBER
diff_hunk: @@ -363,3 +363,33 @@ test('calculateDelayBeforeNextTask() handles one task per round', () => {
   })
   assertEquals(delay, 60_000)
 })
+
+  // Should abort after hitting maxFetchDurationMs and set requestIdleTimeout = true
+test('fetchCAR triggers timeout after long retrieval', async () => {
+  const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))
+  const fetch = async (_url, { signal }) => {
+    return {
+      status: 200,
+      ok: true,
+      body: (async function * () {
+        while (true) {
+          if (signal.aborted) {
+            throw new DOMException('Aborted', 'AbortError')
+          }
+          yield new Uint8Array([0])
+          await sleep(500)
+        }
+      })()
+    }
+  }
+
+  const spark = new Spark({ fetch })
+  const stats = newStats()
+
+  await spark.fetchCAR('http', '/dns/example.com/tcp/80/http', KNOWN_CID, stats, {
+    maxFetchDurationMs: 1000
path: test/spark.js
user: login: juliangruber
body: ```suggestion
    maxFetchDurationMs: 0
```

This way the test finishes sooner
created_at: 2025-03-26T12:48:08Z
author_association: MEMBER

PR comments:
user: login: bajtos
created_at: 2025-03-24T15:45:50Z
body: Thank you @Hany-Almnaem for the pull request! ❤️ 
author_association: MEMBER
user: login: Hany-Almnaem
created_at: 2025-03-24T16:19:06Z
body: @juliangruber, @bajtos 
Thanks for the helpful feedback.
I’ve applied all the suggested changes — everything should be updated now.

Also, I'd love to join if there’s any public channel (Slack, etc.) where I could ask a few technical questions about related repositories.
author_association: CONTRIBUTOR
user: login: bajtos
created_at: 2025-03-25T16:49:41Z
body: >Also, I'd love to join if there’s any public channel (Slack, etc.) where I could ask a few technical questions about related repositories.

You can find us in `#spark` in the Filecoin Slack workspace ([invite link](https://filecoin.io/slack/))
author_association: MEMBER
user: login: juliangruber
created_at: 2025-03-26T14:02:10Z
body: Great! 😍 

Now just need to resolve conflicts, then we can merge and publish
author_association: MEMBER
user: login: juliangruber
created_at: 2025-03-27T09:06:31Z
body: Thank you @Hany-Almnaem 👏 

https://github.com/CheckerNetwork/spark-checker/releases/tag/v1.19.0
author_association: MEMBER