Repository: cds-snc/data-lake
Pull request #251: feat: create anomaly detection alarms for GC Forms
Author: login: patheard
Created: 2025-05-13T20:13:45Z | Merged: 2025-05-14T15:36:43Z

# Summary
Add anomaly detection alarms for all nightly ETLs run against the `Platform / GC Forms` datasets and a sampling of `Platform / GC Notify` datasets.

## ℹ️  Notes
1. These alarms will only be created in Prod.
2. This PR should merge after the Release Please setup PR has merged:
   - #250

# Related
- https://github.com/cds-snc/platform-core-services/issues/691


Diff:
diff --git a/terragrunt/aws/alarms/alarms.tf b/terragrunt/aws/alarms/alarms.tf
index 7c7a24f8..3e123512 100644
--- a/terragrunt/aws/alarms/alarms.tf
+++ b/terragrunt/aws/alarms/alarms.tf
@@ -84,33 +84,43 @@ resource "aws_cloudwatch_metric_alarm" "glue_job_failures" {
   }
 }
 
-resource "aws_cloudwatch_metric_alarm" "platform_gc_forms_etl_user_processed_record_anomaly" {
-  alarm_name          = "platform-gc-forms-etl-user-processed-record-anomaly"
-  alarm_description   = "'Platform / GC Forms' ETL job anomaly detection for the 'user' dataset."
+resource "aws_cloudwatch_metric_alarm" "anomaly_detection" {
+  for_each = var.env == "production" ? tomap({
+    for alarm in local.anomaly_detection_alarms :
+    alarm.name => {
+      metric_name        = alarm.metric_name
+      namespace          = alarm.namespace
+      dataset            = alarm.dataset
+      standard_deviation = alarm.standard_deviation
+    }
+  }) : tomap({})
+
+  alarm_name          = "anomaly-${each.key}"
+  alarm_description   = "Anomaly detection for '${each.value.namespace} / ${each.value.dataset} / ${each.value.metric_name}'"
   comparison_operator = "LessThanLowerOrGreaterThanUpperThreshold"
-  threshold_metric_id = "processed_records_expected"
+  threshold_metric_id = "expected_value"
   evaluation_periods  = 1
   treat_missing_data  = "notBreaching"
 
   metric_query {
-    id          = "processed_records_expected"
-    expression  = "ANOMALY_DETECTION_BAND(processed_records, 0.5)" # standard deviations
-    label       = "Processed Records (Expected)"
+    id          = "expected_value"
+    expression  = "ANOMALY_DETECTION_BAND(actual_value, ${each.value.standard_deviation})"
+    label       = "${each.value.metric_name} (Expected)"
     return_data = "true"
   }
 
   metric_query {
-    id          = "processed_records"
+    id          = "actual_value"
     return_data = "true"
     metric {
-      metric_name = "ProcessedRecordCount"
-      namespace   = "data-lake/etl/gc-forms"
+      metric_name = each.value.metric_name
+      namespace   = each.value.namespace
       period      = 86400 # 1 day
       stat        = "Sum"
       unit        = "Count"
 
       dimensions = {
-        Dataset = "processed-data/user"
+        Dataset = each.value.dataset
       }
     }
   }
diff --git a/terragrunt/aws/alarms/locals.tf b/terragrunt/aws/alarms/locals.tf
index d387934e..3923562b 100644
--- a/terragrunt/aws/alarms/locals.tf
+++ b/terragrunt/aws/alarms/locals.tf
@@ -1,4 +1,56 @@
 locals {
+  anomaly_detection_alarms = [
+    {
+      name               = "platform-gc-forms-etl-submissions-processed-record"
+      namespace          = "data-lake/etl/gc-forms"
+      dataset            = "processed-data/submissions"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    },
+    {
+      name               = "platform-gc-forms-etl-template-processed-record"
+      namespace          = "data-lake/etl/gc-forms"
+      dataset            = "processed-data/template"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    },
+    {
+      name               = "platform-gc-forms-etl-template-to-user-processed-record"
+      namespace          = "data-lake/etl/gc-forms"
+      dataset            = "processed-data/templateToUser"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    },
+    {
+      name               = "platform-gc-forms-etl-user-processed-record"
+      namespace          = "data-lake/etl/gc-forms"
+      dataset            = "processed-data/user"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    },
+    {
+      name               = "platform-gc-notify-etl-notification-history-processed-record"
+      namespace          = "data-lake/etl/gc-notify"
+      dataset            = "notification_history"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    },
+    {
+      name               = "platform-gc-notify-etl-notifications-processed-record"
+      namespace          = "data-lake/etl/gc-notify"
+      dataset            = "notifications"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    },
+    {
+      name               = "platform-gc-notify-etl-users-processed-record"
+      namespace          = "data-lake/etl/gc-notify"
+      dataset            = "users"
+      metric_name        = "ProcessedRecordCount"
+      standard_deviation = 2
+    }
+  ]
+
   data_lake_namespace                      = "data-lake"
   glue_crawler_metric_filter_error_pattern = "ERROR"
   glue_crawler_error_metric_name           = "glue-crawler-error"


Reviews:
user: login: yannickbounouar-cds
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T13:53:19Z
user: login: patheard
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-14T14:35:14Z
user: login: yannickbounouar-cds
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T14:51:45Z

Review comments:
path: terragrunt/aws/alarms/locals.tf
user: login: yannickbounouar-cds
body: @patheard  Does this mean an alarm will be triggered at 0.5 / 1 sd away from the expected value? This is very strict, is this desired? I'd suggest at least sd=2
created_at: 2025-05-14T13:53:15Z
author_association: CONTRIBUTOR
path: terragrunt/aws/alarms/locals.tf
user: login: patheard
body: heh this is an example of me not have a strong data background - totally happy to try it with sd=2 and see how the alarms behave
created_at: 2025-05-14T14:35:14Z
author_association: MEMBER

PR comments:
user: login: github-actions[bot]
created_at: 2025-05-14T14:39:28Z
body: ## Staging: alarms 🚨
**✅ &nbsp; Terraform Init:** `success`
**✅ &nbsp; Terraform Validate:** `success`
**✅ &nbsp; Terraform Format:** `success`
**✅ &nbsp; Terraform Plan:** `success`
**✅ &nbsp; Conftest:** `success` 

**⚠️ &nbsp; Warning:** resources will be destroyed by this change!
```terraform
Plan: 0 to add, 0 to change, 1 to destroy
```

<details>
<summary>Show summary</summary>

| CHANGE |                                       NAME                                        |
|--------|-----------------------------------------------------------------------------------|
| delete | `aws_cloudwatch_metric_alarm.platform_gc_forms_etl_user_processed_record_anomaly` |


</details>


<details>
<summary>Show plan</summary>

```terraform
Resource actions are indicated with the following symbols:
  - destroy

Terraform will perform the following actions:

  # aws_cloudwatch_metric_alarm.platform_gc_forms_etl_user_processed_record_anomaly will be destroyed
  # (because aws_cloudwatch_metric_alarm.platform_gc_forms_etl_user_processed_record_anomaly is not in configuration)
  - resource "aws_cloudwatch_metric_alarm" "platform_gc_forms_etl_user_processed_record_anomaly" {
      - actions_enabled                       = true -> null
      - alarm_actions                         = [] -> null
      - alarm_description                     = "'Platform / GC Forms' ETL job anomaly detection for the 'user' dataset." -> null
      - alarm_name                            = "platform-gc-forms-etl-user-processed-record-anomaly" -> null
      - arn                                   = "arn:aws:cloudwatch:ca-central-1:454671348950:alarm:platform-gc-forms-etl-user-processed-record-anomaly" -> null
      - comparison_operator                   = "LessThanLowerOrGreaterThanUpperThreshold" -> null
      - datapoints_to_alarm                   = 0 -> null
      - dimensions                            = {} -> null
      - evaluation_periods                    = 1 -> null
      - id                                    = "platform-gc-forms-etl-user-processed-record-anomaly" -> null
      - insufficient_data_actions             = [] -> null
      - ok_actions                            = [] -> null
      - period                                = 0 -> null
      - tags                                  = {} -> null
      - tags_all                              = {
          - "CostCentre" = "PlatformDataLake"
          - "Terraform"  = "true"
        } -> null
      - threshold                             = 0 -> null
      - threshold_metric_id                   = "processed_records_expected" -> null
      - treat_missing_data                    = "notBreaching" -> null
        # (6 unchanged attributes hidden)

      - metric_query {
          - id          = "processed_records" -> null
          - period      = 0 -> null
          - return_data = true -> null
            # (3 unchanged attributes hidden)

          - metric {
              - dimensions  = {
                  - "Dataset" = "processed-data/user"
                } -> null
              - metric_name = "ProcessedRecordCount" -> null
              - namespace   = "data-lake/etl/gc-forms" -> null
              - period      = 86400 -> null
              - stat        = "Sum" -> null
              - unit        = "Count" -> null
            }
        }
      - metric_query {
          - expression  = "ANOMALY_DETECTION_BAND(processed_records, 0.5)" -> null
          - id          = "processed_records_expected" -> null
          - label       = "Processed Records (Expected)" -> null
          - period      = 0 -> null
          - return_data = true -> null
            # (1 unchanged attribute hidden)
        }
    }

Plan: 0 to add, 0 to change, 1 to destroy.

─────────────────────────────────────────────────────────────────────────────

Saved the plan to: plan.tfplan

To perform exactly these actions, run the following command to apply:
    terraform apply "plan.tfplan"

```

</details>

<details>
<summary>Show Conftest results</summary>

```sh
WARN - plan.json - main - Missing Common Tags: ["aws_cloudwatch_event_rule.glue_job_failure"]
WARN - plan.json - main - Missing Common Tags: ["aws_cloudwatch_metric_alarm.glue_crawler_error"]
WARN - plan.json - main - Missing Common Tags: ["aws_cloudwatch_metric_alarm.glue_etl_error"]
WARN - plan.json - main - Missing Common Tags: ["aws_cloudwatch_metric_alarm.glue_job_failures"]
WARN - plan.json - main - Missing Common Tags: ["aws_kms_key.cloudwatch"]
WARN - plan.json - main - Missing Common Tags: ["aws_sns_topic.cloudwatch_alarm_action"]
WARN - plan.json - main - Missing Common Tags: ["aws_sns_topic.cloudwatch_ok_action"]

26 tests, 19 passed, 7 warnings, 0 failures, 0 exceptions

```

</details>

author_association: CONTRIBUTOR