Repository: hyperjumptech/monika (ts)

Issue:
Node.js fetch : handle HTTP status code 3xx to comply with --follow-redirects flag

Pull request description:
feat(node.js fetch): #1268 implement redirect for fetch
# Monika Pull Request (PR)

This PR resolves #1268 

## What feature/issue does this PR add

1. Handle HTTP status code 3xx for Node.js fetch implementation

## How did you implement / how did you fix it

1. Manually handle status code 3xx on Node.js fetch function
2. Move HTTP client's dependency to flag to parameter function
3. Add tests

## How to test

1. `npm run test`
2. Create `monika.yml` config file below

```YAML
probes:
  - id: 'Example'
    requests:
      - url: https://httpbin.org/redirect/3
        timeout: 60000
    interval: 5
    incidentThreshold: 1
    recoveryThreshold: 1
    alerts:
      - assertion: response.time >= 60000
        message: HTTP response time is {{ response.time }}, expecting 10000
```

3. `npm run start -- --native-fetch -c monika.yml -r 1`

Patch:
diff --git a/src/components/probe/prober/http/index.ts b/src/components/probe/prober/http/index.ts
index 0d054120..5c0b1e63 100644
--- a/src/components/probe/prober/http/index.ts
+++ b/src/components/probe/prober/http/index.ts
@@ -22,6 +22,7 @@
  * SOFTWARE.                                                                      *
  **********************************************************************************/
 
+import { monikaFlagsDefaultValue } from '../../../../flag'
 import { BaseProber, NotificationType, type ProbeParams } from '..'
 import { getContext } from '../../../../context'
 import events from '../../../../events'
@@ -56,6 +57,11 @@ export class HTTPProber extends BaseProber {
       responses.push(
         // eslint-disable-next-line no-await-in-loop
         await httpRequest({
+          isVerbose: getContext().flags.verbose,
+          maxRedirects:
+            getContext().flags['follow-redirects'] ||
+            monikaFlagsDefaultValue['follow-redirects'],
+          isEnableFetch: getContext().flags['native-fetch'],
           requestConfig: { ...requestConfig, signal },
           responses,
         })
diff --git a/src/components/probe/prober/http/request.ts b/src/components/probe/prober/http/request.ts
index aadca6a3..ceba6741 100644
--- a/src/components/probe/prober/http/request.ts
+++ b/src/components/probe/prober/http/request.ts
@@ -34,13 +34,11 @@ import {
 // eslint-disable-next-line no-restricted-imports
 import * as qs from 'querystring'
 
-import { getContext } from '../../../../context'
 import { icmpRequest } from '../icmp/request'
 import registerFakes from '../../../../utils/fakes'
 import { sendHttpRequest, sendHttpRequestFetch } from '../../../../utils/http'
 import { log } from '../../../../utils/pino'
 import { AxiosError } from 'axios'
-import { MonikaFlags } from 'src/flag'
 import { getErrorMessage } from '../../../../utils/catch-error-handler'
 import { errors as undiciErrors } from 'undici'
 import Joi from 'joi'
@@ -49,8 +47,11 @@ import Joi from 'joi'
 registerFakes(Handlebars)
 
 type probingParams = {
+  maxRedirects: number
   requestConfig: Omit<RequestConfig, 'saveBody' | 'alert'> // is a config object
   responses: Array<ProbeRequestResponse> // an array of previous responses
+  isVerbose?: boolean
+  isEnableFetch?: boolean
 }
 
 const UndiciErrorValidator = Joi.object({
@@ -63,8 +64,11 @@ const UndiciErrorValidator = Joi.object({
  * @returns ProbeRequestResponse, response to the probe request
  */
 export async function httpRequest({
+  maxRedirects,
   requestConfig,
   responses,
+  isEnableFetch,
+  isVerbose,
 }: probingParams): Promise<ProbeRequestResponse> {
   // Compile URL using handlebars to render URLs that uses previous responses data
   const { method, url, headers, timeout, body, ping, allowUnauthorized } =
@@ -72,8 +76,6 @@ export async function httpRequest({
   const newReq = { method, headers, timeout, body, ping }
   const renderURL = Handlebars.compile(url)
   const renderedURL = renderURL({ responses })
-
-  const { flags } = getContext()
   newReq.headers = compileHeaders(headers, body, responses as never)
   // compile body needs to modify headers if necessary
   const { headers: newHeaders, body: newBody } = compileBody(
@@ -97,10 +99,11 @@ export async function httpRequest({
     }
 
     // Do the request using compiled URL and compiled headers (if exists)
-    if (flags['native-fetch']) {
+    if (isEnableFetch) {
       return await probeHttpFetch({
         startTime,
-        flags,
+        isVerbose,
+        maxRedirects,
         renderedURL,
         requestParams: { ...newReq, headers: requestHeaders },
         allowUnauthorized,
@@ -109,7 +112,7 @@ export async function httpRequest({
 
     return await probeHttpAxios({
       startTime,
-      flags,
+      maxRedirects,
       renderedURL,
       requestParams: { ...newReq, headers: requestHeaders },
       allowUnauthorized,
@@ -228,15 +231,17 @@ function compileBody(
 
 async function probeHttpFetch({
   startTime,
-  flags,
   renderedURL,
   requestParams,
   allowUnauthorized,
+  maxRedirects,
+  isVerbose,
 }: {
   startTime: number
-  flags: MonikaFlags
   renderedURL: string
   allowUnauthorized: boolean | undefined
+  maxRedirects: number
+  isVerbose?: boolean
   requestParams: {
     method: string | undefined
     headers: Headers | undefined
@@ -245,13 +250,21 @@ async function probeHttpFetch({
     ping: boolean | undefined
   }
 }): Promise<ProbeRequestResponse> {
-  if (flags.verbose) log.info(`Probing ${renderedURL} with Node.js fetch`)
+  if (isVerbose) log.info(`Probing ${renderedURL} with Node.js fetch`)
+  console.log(
+    'renderedURL =',
+    renderedURL,
+    '| requestParams =',
+    JSON.stringify(requestParams),
+    '| maxRedirects =',
+    maxRedirects
+  )
   const response = await sendHttpRequestFetch({
     ...requestParams,
     allowUnauthorizedSsl: allowUnauthorized,
     keepalive: true,
     url: renderedURL,
-    maxRedirects: flags['follow-redirects'],
+    maxRedirects,
     body:
       typeof requestParams.body === 'string'
         ? requestParams.body
@@ -286,15 +299,15 @@ async function probeHttpFetch({
 
 async function probeHttpAxios({
   startTime,
-  flags,
   renderedURL,
-  requestParams,
   allowUnauthorized,
+  maxRedirects,
+  requestParams,
 }: {
   startTime: number
-  flags: MonikaFlags
   renderedURL: string
   allowUnauthorized: boolean | undefined
+  maxRedirects: number
   requestParams: {
     method: string | undefined
     headers: Headers | undefined
@@ -308,7 +321,7 @@ async function probeHttpAxios({
     allowUnauthorizedSsl: allowUnauthorized,
     keepalive: true,
     url: renderedURL,
-    maxRedirects: flags['follow-redirects'],
+    maxRedirects,
     body:
       typeof requestParams.body === 'string'
         ? requestParams.body
diff --git a/src/utils/http.ts b/src/utils/http.ts
index 1b95e1e4..696ecb68 100644
--- a/src/utils/http.ts
+++ b/src/utils/http.ts
@@ -89,36 +89,85 @@ function convertHeadersToAxios(headersInit: HeadersInit | undefined) {
 export async function sendHttpRequestFetch(
   config: HttpRequestParams
 ): Promise<Response> {
-  const {
-    allowUnauthorizedSsl,
-    body,
-    headers,
-    maxRedirects,
-    method,
-    timeout,
-    url,
-  } = config
+  const { maxRedirects, timeout, url } = config
   const controller = new AbortController()
   const { signal } = controller
   const timeoutId = setTimeout(() => {
     controller.abort()
   }, timeout || DEFAULT_TIMEOUT)
-  const resp = await fetch(url, {
-    body: body === '' ? undefined : body,
-    dispatcher: new Agent({
-      connect: {
-        rejectUnauthorized: !allowUnauthorizedSsl,
-      },
-      maxRedirections: maxRedirects,
-    }),
-    headers,
-    keepalive: true,
-    method,
-    signal,
-  }).then((response) => {
+  const fetcher = compileFetch(config, signal)
+  return fetchRedirect(url, maxRedirects, fetcher).then((response) => {
     clearTimeout(timeoutId)
     return response
   })
+}
+
+function compileFetch(
+  config: HttpRequestParams,
+  signal: AbortSignal
+): (url: string) => Promise<Response> {
+  const { allowUnauthorizedSsl, body, headers, method } = config
+
+  return (url) =>
+    fetch(url, {
+      body: body === '' ? undefined : body,
+      redirect: 'manual',
+      dispatcher: new Agent({
+        connect: {
+          rejectUnauthorized: !allowUnauthorizedSsl,
+        },
+      }),
+      headers,
+      keepalive: true,
+      method,
+      signal,
+    })
+}
+
+// fetchRedirect handles HTTP status code 3xx returned by fetcher
+async function fetchRedirect(
+  url: string,
+  maxRedirects: number | undefined,
+  fetcher: (url: string) => Promise<Response>
+) {
+  let redirected = 0
+  let currentResponse: Response
+  let nextUrl = url
+
+  // do HTTP fetch request at least once
+  // then follow redirect based maxRedirects value and HTTP status code 3xx
+  do {
+    // eslint-disable-next-line no-await-in-loop
+    currentResponse = await fetcher(nextUrl)
+    // check for HTTP status code 3xx
+    const shouldRedirect =
+      currentResponse.status >= 300 && currentResponse.status < 400
+    if (!shouldRedirect) break
+
+    // location header could either be full url, relative path, or absolute path
+    // e.g. "https://something.tld", "new/path", "/new/path", respectively
+    // refer to : RFC-7231 https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.2
+    const newLocation = currentResponse.headers.get('location') || ''
+    // try-catch to evaluate if redirect location is a url
+    try {
+      // when it is valid url, immediately set nextUrl from location header
+      nextUrl = new URL(
+        currentResponse.headers.get('location') || ''
+      ).toString()
+    } catch {
+      // new redirect location is relative / absolute url
+      const newEndpoint = newLocation.startsWith('/')
+        ? newLocation
+        : `/${newLocation}`
+      // parse nextUrl to Node.js URL to get protocol and host
+      const parsedURL = new URL(nextUrl)
+      // compose nextUrl from parsed protocol, host, and newEndpoint
+      nextUrl = `${parsedURL.protocol}//${parsedURL.host}${newEndpoint}`
+    }
+
+    // increment redirected value for while loop
+    redirected++
+  } while (redirected <= (maxRedirects || 0))
 
-  return resp
+  return currentResponse
 }


Test patch:
diff --git a/src/components/probe/prober/http/request.test.ts b/src/components/probe/prober/http/request.test.ts
index 8d0f193a..c5367796 100644
--- a/src/components/probe/prober/http/request.test.ts
+++ b/src/components/probe/prober/http/request.test.ts
@@ -91,6 +91,7 @@ describe('probingHTTP', () => {
           try {
             // eslint-disable-next-line no-await-in-loop
             const resp = await httpRequest({
+              maxRedirects: 0,
               requestConfig: request,
               responses,
             })
@@ -138,6 +139,7 @@ describe('probingHTTP', () => {
       }
 
       const result = await httpRequest({
+        maxRedirects: 0,
         requestConfig: request,
         responses: [],
       })
@@ -176,6 +178,7 @@ describe('probingHTTP', () => {
 
       // act
       const res = await httpRequest({
+        maxRedirects: 0,
         requestConfig: request,
         responses: [],
       })
@@ -218,6 +221,7 @@ describe('probingHTTP', () => {
 
       // act
       const res = await httpRequest({
+        maxRedirects: 0,
         requestConfig: request,
         responses: [],
       })
@@ -260,6 +264,7 @@ describe('probingHTTP', () => {
 
       // act
       const res = await httpRequest({
+        maxRedirects: 0,
         requestConfig: request,
         responses: [],
       })
@@ -302,6 +307,7 @@ describe('probingHTTP', () => {
 
       // act
       const res = await httpRequest({
+        maxRedirects: 0,
         requestConfig: request,
         responses: [],
       })
@@ -310,6 +316,93 @@ describe('probingHTTP', () => {
       expect(res.status).to.eq(200)
     })
 
+    it('Should handle HTTP redirect with axios', async () => {
+      // arrange
+      server.use(
+        http.get(
+          'https://example.com/get',
+          () => new HttpResponse(null, { status: 200 })
+        ),
+        http.get(
+          'https://example.com/redirect/:nredirect',
+          async ({ params }) => {
+            const { nredirect } = params
+            const castRedirect = Number(nredirect)
+            return new HttpResponse(null, {
+              status: 302,
+              headers: [
+                [
+                  'Location',
+                  castRedirect === 1
+                    ? 'https://example.com/get'
+                    : `https://example.com/redirect/${castRedirect - 1}`,
+                ],
+              ],
+            })
+          }
+        )
+      )
+
+      const requestConfig: RequestConfig = {
+        url: 'https://example.com/redirect/3',
+        method: 'GET',
+        body: '',
+        timeout: 10_000,
+      }
+
+      const res = await httpRequest({
+        isEnableFetch: true,
+        maxRedirects: 3,
+        requestConfig,
+        responses: [],
+      })
+
+      expect(res.status).to.eq(200)
+    })
+    it('Should handle HTTP redirect with fetch', async () => {
+      // arrange
+      server.use(
+        http.get(
+          'https://example.com/get',
+          () => new HttpResponse(null, { status: 200 })
+        ),
+        http.get(
+          'https://example.com/redirect/:nredirect',
+          async ({ params }) => {
+            const { nredirect } = params
+            const castRedirect = Number(nredirect)
+            return new HttpResponse(null, {
+              status: 302,
+              headers: [
+                [
+                  'Location',
+                  castRedirect === 1
+                    ? 'https://example.com/get'
+                    : `https://example.com/redirect/${castRedirect - 1}`,
+                ],
+              ],
+            })
+          }
+        )
+      )
+
+      const requestConfig: RequestConfig = {
+        url: 'https://example.com/redirect/3',
+        method: 'GET',
+        body: '',
+        timeout: 10_000,
+      }
+
+      const res = await httpRequest({
+        isEnableFetch: false,
+        maxRedirects: 3,
+        requestConfig,
+        responses: [],
+      })
+
+      expect(res.status).to.eq(200)
+    })
+
     it('should send request with text-plain content-type even with allowUnauthorized option', async () => {
       // arrange
       server.use(
@@ -345,6 +438,7 @@ describe('probingHTTP', () => {
 
       // act
       const res = await httpRequest({
+        maxRedirects: 0,
         requestConfig: request,
         responses: [],
       })