Repository: hyperjumptech/monika
Pull request #1184 (2023-11-15 11:54:17)

Fix: Send Recovery Notification from Previous Incident Before Monika Restart in Symon Mode
# Monika Pull Request (PR)

## What feature/issue does this PR add

Send recovery notification from previous incident before Monika restart in Symon mode. 

## How did you implement / how did you fix it

- Add some tests.
- Receive last event alert id data from Symon.
- Construct probe incident state from last event data.

## How to test

1. Make an incident probe.
2. Stop Monika.
3. Make the probe target healthy.
4. Run Monika.


## Video

https://github.com/hyperjumptech/monika/assets/15191978/d51f5464-1cc8-4678-8d77-27ddb18916bd


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/hyperjumptech/monika/pull/1184?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech) Report
Attention: `10 lines` in your changes are missing coverage. Please review.
> Comparison is base [(`bfd80b2`)](https://app.codecov.io/gh/hyperjumptech/monika/commit/bfd80b206b9641d79a44ae4366ef80cb819816f2?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech) 61.05% compared to head [(`ff2da47`)](https://app.codecov.io/gh/hyperjumptech/monika/pull/1184?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech) 61.91%.

> :exclamation: Current head ff2da47 differs from pull request most recent head 46304d0. Consider uploading reports for the commit 46304d0 to get more accurate results

| [Files](https://app.codecov.io/gh/hyperjumptech/monika/pull/1184?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech) | Patch % | Lines |
|---|---|---|
| [src/components/probe/prober/index.ts](https://app.codecov.io/gh/hyperjumptech/monika/pull/1184?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech#diff-c3JjL2NvbXBvbmVudHMvcHJvYmUvcHJvYmVyL2luZGV4LnRz) | 71.42% | [5 Missing and 5 partials :warning: ](https://app.codecov.io/gh/hyperjumptech/monika/pull/1184?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #1184      +/-   ##
==========================================
+ Coverage   61.05%   61.91%   +0.85%     
==========================================
  Files         107      107              
  Lines        3133     3169      +36     
  Branches      531      541      +10     
==========================================
+ Hits         1913     1962      +49     
+ Misses       1037     1020      -17     
- Partials      183      187       +4     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/hyperjumptech/monika/pull/1184?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperjumptech).

kevinhermawan: It works as expected, thanks!

Patch:
diff --git a/src/components/config/validation/validator/probe.ts b/src/components/config/validation/validator/probe.ts
index 28ba084b..44bc36ee 100644
--- a/src/components/config/validation/validator/probe.ts
+++ b/src/components/config/validation/validator/probe.ts
@@ -160,6 +160,7 @@ export async function validateProbes(probes: Probe[]): Promise<Probe[]> {
             .min(1),
           interval: joi.number().default(DEFAULT_INTERVAL).min(1),
           lastEvent: joi.object({
+            alertId: joi.string(),
             createdAt: joi.string().allow(''),
             recoveredAt: joi.string().allow('', null),
           }),
diff --git a/src/components/downtime-counter/index.ts b/src/components/downtime-counter/index.ts
index 8422efce..365f7f6f 100644
--- a/src/components/downtime-counter/index.ts
+++ b/src/components/downtime-counter/index.ts
@@ -30,10 +30,12 @@ type DowntimeCounter = {
   alert: ProbeAlert
   probeID: string
   url: string
+  createdAt?: Date
 }
 
 export function startDowntimeCounter({
   alert,
+  createdAt,
   probeID,
   url,
 }: DowntimeCounter): void {
@@ -41,7 +43,7 @@ export function startDowntimeCounter({
     alert,
     probeID,
     probeRequestURL: url,
-    createdAt: new Date(),
+    createdAt: createdAt || new Date(),
   }
 
   setContext({ incidents: [...getContext().incidents, newIncident] })
diff --git a/src/components/probe/prober/index.ts b/src/components/probe/prober/index.ts
index 48ea34a8..aa19de48 100644
--- a/src/components/probe/prober/index.ts
+++ b/src/components/probe/prober/index.ts
@@ -29,6 +29,7 @@ import type { Probe, ProbeAlert } from '../../../interfaces/probe'
 import {
   probeRequestResult,
   type ProbeRequestResponse,
+  type RequestConfig,
 } from '../../../interfaces/request'
 import { FAILED_REQUEST_ASSERTION } from '../../../looper'
 import type { ValidatedResponse } from '../../../plugins/validate-response'
@@ -91,6 +92,8 @@ export class BaseProber implements Prober {
     this.counter = counter
     this.notifications = notifications
     this.probeConfig = probeConfig
+
+    this.initializeProbeState()
   }
 
   async probe(incidentRetryAttempt: number): Promise<void> {
@@ -414,6 +417,95 @@ export class BaseProber implements Prober {
     log.warn(`${this.getMessagePrefix()} ${message.filter(Boolean).join(', ')}`)
   }
 
+  private initializeProbeState() {
+    if (
+      !this.probeConfig?.lastEvent ||
+      this.probeConfig.lastEvent?.recoveredAt !== null
+    ) {
+      return
+    }
+
+    if (!this.probeConfig.lastEvent?.alertId) {
+      log.error(`Last event ID in probe ${this.probeConfig.id} is required`)
+      return
+    }
+
+    const { alertId, createdAt } = this.probeConfig.lastEvent
+    const alert = this.getAlerts().find(({ id }) => id === alertId)
+
+    if (!alert) {
+      log.error(
+        `Alert ID: ${alertId} is not found in probe ${this.probeConfig.id}`
+      )
+      return
+    }
+
+    const request = this.getRequestByAlertId(alertId)
+    if (!request) {
+      log.error(
+        `Request for alert ID: ${alertId} is not found in probe ${this.probeConfig.id}`
+      )
+      return
+    }
+
+    startDowntimeCounter({
+      alert,
+      probeID: this.probeConfig.id,
+      url: request.url,
+      createdAt,
+    })
+  }
+
+  private getAlerts() {
+    const httpAlerts =
+      this.probeConfig?.requests?.map(({ alerts }) => alerts).find(Boolean) ||
+      []
+    const mariadbAlerts =
+      this.probeConfig?.mariadb?.map(({ alerts }) => alerts).find(Boolean) || []
+    const mongoAlerts =
+      this.probeConfig?.mongo?.map(({ alerts }) => alerts).find(Boolean) || []
+    const mysqlAlerts =
+      this.probeConfig?.mysql?.map(({ alerts }) => alerts).find(Boolean) || []
+    const pingAlerts =
+      this.probeConfig?.ping?.map(({ alerts }) => alerts).find(Boolean) || []
+    const postgresAlerts =
+      this.probeConfig?.postgres?.map(({ alerts }) => alerts).find(Boolean) ||
+      []
+    const redisAlerts =
+      this.probeConfig?.redis?.map(({ alerts }) => alerts).find(Boolean) || []
+    const socketAlerts = this.probeConfig?.socket?.alerts || []
+
+    return [
+      ...(this.probeConfig?.alerts || []),
+      ...httpAlerts,
+      ...mariadbAlerts,
+      ...mongoAlerts,
+      ...mysqlAlerts,
+      ...pingAlerts,
+      ...postgresAlerts,
+      ...redisAlerts,
+      ...socketAlerts,
+    ]
+  }
+
+  private getRequestByAlertId(alertId: string): RequestConfig | null {
+    if (!this.probeConfig.requests) {
+      return null
+    }
+
+    for (const request of this.probeConfig.requests) {
+      if (!request?.alerts) {
+        continue
+      }
+
+      if (request.alerts.some(({ id }) => id === alertId)) {
+        return request
+      }
+    }
+
+    return null
+  }
+
   private hasNotification() {
     return this.notifications.length > 0
   }
diff --git a/src/interfaces/probe.ts b/src/interfaces/probe.ts
index 4a269dd7..bdbb4bc6 100644
--- a/src/interfaces/probe.ts
+++ b/src/interfaces/probe.ts
@@ -103,7 +103,8 @@ export interface Probe {
   recoveryThreshold?: number
   alerts: ProbeAlert[]
   lastEvent?: {
+    alertId?: string
     createdAt?: Date
-    recoveredAt?: Date
+    recoveredAt?: Date | null
   }
 }


Test patch:
diff --git a/src/components/config/validation/validator/probe.test.ts b/src/components/config/validation/validator/probe.test.ts
index 231604b0..3f16ed0c 100644
--- a/src/components/config/validation/validator/probe.test.ts
+++ b/src/components/config/validation/validator/probe.test.ts
@@ -28,8 +28,8 @@ import type { Probe } from '../../../../interfaces/probe'
 
 import { FAILED_REQUEST_ASSERTION } from '../../../../looper'
 import { validateProbes } from './probe'
-import { resetContext, setContext } from '../../../../../src/context'
-import type { MonikaFlags } from '../../../../../src/flag'
+import { resetContext, setContext } from '../../../../context'
+import type { MonikaFlags } from '../../../../flag'
 
 describe('Probe validation', () => {
   describe('Probe sanitization', () => {
diff --git a/src/components/probe/prober/index.test.ts b/src/components/probe/prober/index.test.ts
new file mode 100644
index 00000000..7f209fac
--- /dev/null
+++ b/src/components/probe/prober/index.test.ts
@@ -0,0 +1,413 @@
+/**********************************************************************************
+ * MIT License                                                                    *
+ *                                                                                *
+ * Copyright (c) 2021 Hyperjump Technology                                        *
+ *                                                                                *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy   *
+ * of this software and associated documentation files (the "Software"), to deal  *
+ * in the Software without restriction, including without limitation the rights   *
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      *
+ * copies of the Software, and to permit persons to whom the Software is          *
+ * furnished to do so, subject to the following conditions:                       *
+ *                                                                                *
+ * The above copyright notice and this permission notice shall be included in all *
+ * copies or substantial portions of the Software.                                *
+ *                                                                                *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     *
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       *
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    *
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         *
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  *
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  *
+ * SOFTWARE.                                                                      *
+ **********************************************************************************/
+
+import { rest } from 'msw'
+import { setupServer } from 'msw/node'
+import { expect } from '@oclif/test'
+
+import type { ProberMetadata } from '.'
+
+import { getContext } from '../../../context'
+import { createProber } from './factory'
+
+describe('Prober', () => {
+  describe('Initial incident state', () => {
+    let probeRequestTotal = 0
+    let webhookBody: Record<string, string> | null = null
+    const server = setupServer(
+      rest.get('https://example.com', (_, res, ctx) => {
+        probeRequestTotal++
+        return res(ctx.status(200))
+      }),
+      rest.post('https://example.com/webhook', async (req, res, ctx) => {
+        webhookBody = await req.json()
+        return res(ctx.status(202))
+      })
+    )
+
+    before(() => {
+      server.listen()
+    })
+    afterEach(() => {
+      server.resetHandlers()
+      probeRequestTotal = 0
+      webhookBody = null
+    })
+    after(() => {
+      server.close()
+    })
+
+    it('should not initialize probe state if last event is not specify', async () => {
+      // arrange
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          requests: [{ body: '', timeout: 30_000, url: 'https://example.com' }],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).eq(null)
+      expect(getContext().incidents.length).eq(0)
+    })
+
+    it('should not initialize probe state if last event id is not specify', async () => {
+      // arrange
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            createdAt: new Date(),
+            recoveredAt: null,
+          },
+          requests: [{ body: '', timeout: 30_000, url: 'https://example.com' }],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).eq(null)
+      expect(getContext().incidents.length).eq(0)
+    })
+
+    it('should not initialize probe state if last event recovered at is not null', async () => {
+      // arrange
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            alertId: 'iCzgj',
+            createdAt: new Date(),
+            recoveredAt: new Date(),
+          },
+          requests: [{ body: '', timeout: 30_000, url: 'https://example.com' }],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).eq(null)
+      expect(getContext().incidents.length).eq(0)
+    })
+
+    it('should not initialize probe state if alert is not found', async () => {
+      // arrange
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            alertId: 'iCzgj',
+            createdAt: new Date(),
+            recoveredAt: null,
+          },
+          requests: [{ body: '', timeout: 30_000, url: 'https://example.com' }],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).eq(null)
+      expect(getContext().incidents.length).eq(0)
+    })
+
+    it('should send recovery notification if recovered_at is null and target is healthy', async () => {
+      // arrange
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            alertId: 'iCzgj',
+            createdAt: new Date(),
+            recoveredAt: null,
+          },
+          recoveryThreshold: 1,
+          requests: [
+            {
+              body: '',
+              timeout: 30_000,
+              url: 'https://example.com',
+              alerts: [
+                {
+                  id: 'iCzgj',
+                  assertion: 'response.status != 200',
+                  message: 'Server is down!',
+                },
+              ],
+            },
+          ],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+      await sleep(200)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).not.eq(null)
+      expect(getContext().incidents.length).eq(0)
+    })
+
+    it('should not send incident notification if recovered_at is null and target is not healthy', async () => {
+      // arrange
+      server.use(
+        rest.get('https://example.com', (_, res, ctx) => {
+          probeRequestTotal++
+          return res(ctx.status(404))
+        })
+      )
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            alertId: 'iCzgj',
+            createdAt: new Date(),
+            recoveredAt: null,
+          },
+          incidentThreshold: 1,
+          recoveryThreshold: 1,
+          requests: [
+            {
+              body: '',
+              timeout: 30_000,
+              url: 'https://example.com',
+              alerts: [
+                {
+                  id: 'iCzgj',
+                  assertion: 'response.status != 200',
+                  message: 'Server is down!',
+                },
+              ],
+            },
+          ],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).eq(null)
+      expect(getContext().incidents.length).eq(1)
+    })
+
+    it('should not send recovery notification if recovered_at is not null and target is healthy', async () => {
+      // arrange
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            alertId: 'iCzgj',
+            createdAt: new Date(),
+            recoveredAt: new Date(),
+          },
+          recoveryThreshold: 1,
+          requests: [
+            {
+              body: '',
+              timeout: 30_000,
+              url: 'https://example.com',
+              alerts: [
+                {
+                  id: 'iCzgj',
+                  assertion: 'response.status != 200',
+                  message: 'Server is down!',
+                },
+              ],
+            },
+          ],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).eq(null)
+      expect(getContext().incidents.length).eq(0)
+    })
+
+    it('should send incident notification if recovered_at is not null and target is not healthy', async () => {
+      // arrange
+      server.use(
+        rest.get('https://example.com', (_, res, ctx) => {
+          probeRequestTotal++
+          return res(ctx.status(404))
+        })
+      )
+      const proberMetadata: ProberMetadata = {
+        counter: 0,
+        notifications: [
+          {
+            id: 'NMEdP',
+            type: 'webhook',
+            data: { url: 'https://example.com/webhook' },
+          },
+        ],
+        probeConfig: {
+          alerts: [],
+          id: 'bcDnX',
+          interval: 2,
+          name: 'Example',
+          lastEvent: {
+            alertId: 'iCzgj',
+            createdAt: new Date(),
+            recoveredAt: new Date(),
+          },
+          incidentThreshold: 1,
+          recoveryThreshold: 1,
+          requests: [
+            {
+              body: '',
+              timeout: 30_000,
+              url: 'https://example.com',
+              alerts: [
+                {
+                  id: 'iCzgj',
+                  assertion: 'response.status != 200',
+                  message: 'Server is down!',
+                },
+              ],
+            },
+          ],
+        },
+      }
+
+      // act
+      const prober = createProber(proberMetadata)
+      await prober.probe(0)
+      await sleep(200)
+
+      // assert
+      expect(probeRequestTotal).eq(1)
+      expect(webhookBody).not.eq(null)
+      expect(getContext().incidents.length).eq(1)
+    })
+  })
+})
+
+function sleep(durationMs: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, durationMs)
+  })
+}