Repository: prometheus-community/yet-another-cloudwatch-exporter (Go)

Issue:
EC2 discovery with searchTags does not work
Simply this job just does not work correctly...

```YAML
config:
  discovery:
    exportedTagsOnMetrics:
      ec2:
        - Name
        - cluster
        - service
        - env
    jobs:
    - type: AWS/EC2
      enableMetricData: true
      regions:
        - us-west-2
      searchTags:
        - Key: InstanceFamily
          Value: ^(t2|t3)$
      awsDimensions:
        - InstanceId
      metrics:
        - name: CPUCreditUsage
          statistics:
          - Average
          period: 300
          length: 3600
        - name: CPUCreditBalance
          statistics:
          - Average
          period: 300
          length: 3600
```

returned metrics:
```
# HELP aws_ec2_cpucredit_balance_average Help is not implemented yet.
# TYPE aws_ec2_cpucredit_balance_average gauge
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc05-dev-bastion-ASG",name="global",region="us-west-2"} 864
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc05-v16-master_node-ASG",name="global",region="us-west-2"} 864
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-test-bastion-ASG",name="global",region="us-west-2"} 72
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-compute_worker_node-ASG",name="global",region="us-west-2"} 576
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-master_node-ASG",name="global",region="us-west-2"} 864
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-storage_worker_node-ASG",name="global",region="us-west-2"} 576
# HELP aws_ec2_cpucredit_usage_average Help is not implemented yet.
# TYPE aws_ec2_cpucredit_usage_average gauge
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc05-dev-bastion-ASG",name="global",region="us-west-2"} 0.027776
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc05-v16-master_node-ASG",name="global",region="us-west-2"} 1.0788769999999999
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-test-bastion-ASG",name="global",region="us-west-2"} 0.035522
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-compute_worker_node-ASG",name="global",region="us-west-2"} 0.358226
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-master_node-ASG",name="global",region="us-west-2"} 0.90029
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-storage_worker_node-ASG",name="global",region="us-west-2"} 0.336893
```

even trying forcing dimension (looks like this option was removed or not ? documentation sucks here)

```YAML
discovery:
    exportedTagsOnMetrics:
      ec2:
        - Name
        - cluster
        - service
        - env
    jobs:
    - type: AWS/EC2
      enableMetricData: true
      regions:
        - us-west-2
      searchTags:
        - Key: InstanceFamily
          Value: ^(t2|t3)$
      awsDimensions:
        - InstanceId
      metrics:
        - name: CPUCreditUsage
          statistics:
          - Average
          period: 300
          length: 3600
        - name: CPUCreditBalance
          statistics:
          - Average
          period: 300
          length: 3600
```

it still the same. It only works when searchTags are removed, but then `exportedTagsOnMetrics` option is not honored


```# HELP aws_ec2_cpucredit_balance_average Help is not implemented yet.
# TYPE aws_ec2_cpucredit_balance_average gauge
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxx",dimension_AutoScalingGroupName="",dimension_InstanceId="i-xxxxxxxxxxxxxxxxxxxx",name="arn:aws:ec2:us-west-2:xxxxxxxxxxxx:instance/i-xxxxxxxxxxxxxxxxxxxx",region="us-west-2"} 1296
aws_ec2_cpucredit_balance_average{account_id="xxxxxxxxxxxx",dimension_AutoScalingGroupName="dc05-dev-bastion-ASG",dimension_InstanceId="",name="global",region="us-west-2"} 864

# HELP aws_ec2_cpucredit_usage_average Help is not implemented yet.
# TYPE aws_ec2_cpucredit_usage_average gauge
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxx",dimension_AutoScalingGroupName="",dimension_InstanceId="i-xxxxxxxxxxxxxxxxxxxx",name="arn:aws:ec2:us-west-2:xxxxxxxxxxxx:instance/i-xxxxxxxxxxxxxxxxxxxx",region="us-west-2"} 0.572161
aws_ec2_cpucredit_usage_average{account_id="xxxxxxxxxxxx",dimension_AutoScalingGroupName="dc13-v16-storage_worker_node-ASG",dimension_InstanceId="",name="global",region="us-west-2"} 0.333155

# HELP aws_ec2_info Help is not implemented yet.
# TYPE aws_ec2_info gauge
aws_ec2_info{name="arn:aws:ec2:us-west-2:xxxxxxxxxxxx:instance/i-0007435c3f6c39c0f",tag_Name="dc13-v16-airflow_worker_node",tag_xxxxxxxx_cluster="dc13",tag_xxxxxxxx_env="v16",tag_xxxxxxxx_service="kubernetes_airflow_worker",tag_aws_autoscaling_groupName="dc13-v16-airflow_worker-ASG",tag_cluster="dc13",tag_cluster_name="",tag_env="v16",tag_inventory="",tag_k8s_version="v16",tag_metrics_etcd_port="",tag_metrics_etcd_scrape="",tag_metrics_mongodb_port="",tag_metrics_mongodb_scrape="",tag_metrics_mysql_port="",tag_metrics_mysql_scrape="",tag_metrics_node_port="",tag_metrics_node_scrape="",tag_metrics_rabbitmq_port="",tag_metrics_rabbitmq_scrape="",tag_metrics_redis_port="",tag_metrics_redis_scrape="",tag_node_number="",tag_node_role="airflow",tag_node_schedulable="true",tag_node_type="worker",tag_reference="",tag_role="",tag_service="",tag_service_group="",tag_service_mongo="",tag_service_mongo_replica=""} 0
```

looks like TAGs are added as matrix of all possibilities across all instances 🤦 


Discussion:

Patch:
diff --git a/pkg/aws_cloudwatch.go b/pkg/aws_cloudwatch.go
index 745449d19..3a6654ebc 100644
--- a/pkg/aws_cloudwatch.go
+++ b/pkg/aws_cloudwatch.go
@@ -307,6 +307,12 @@ func getFilteredMetricDatas(region string, accountId *string, namespace string,
 		if len(dimensionNameList) > 0 && !metricDimensionsMatchNames(cwMetric, dimensionNameList) {
 			continue
 		}
+
+		/**
+			This loop takes a list of dimensions for an individual metric returned from AWS ResourceGroupsTaggingApi#GetResources.
+			It filters those dimensions against a user-supplied list of dimensions by name and value, and if they match,
+			adds the metric to a list of metrics to have its values retrieved.
+		*/
 		for _, dimension := range cwMetric.Dimensions {
 			if dimensionFilterValues, ok := dimensionsFilter[*dimension.Name]; ok {
 				if d, ok := dimensionFilterValues[*dimension.Value]; !ok {
@@ -318,8 +324,12 @@ func getFilteredMetricDatas(region string, accountId *string, namespace string,
 					alreadyFound = true
 					r = d
 				}
+			} else {
+				skip = true
+				break
 			}
 		}
+
 		if !skip {
 			for _, stats := range m.Statistics {
 				id := fmt.Sprintf("id_%d", rand.Int())
diff --git a/pkg/aws_tags.go b/pkg/aws_tags.go
index 1a4d452e3..8b36edee1 100644
--- a/pkg/aws_tags.go
+++ b/pkg/aws_tags.go
@@ -119,7 +119,7 @@ func (iface tagsInterface) get(ctx context.Context, job *Job, region string) ([]
 				if resource.filterThroughTags(job.SearchTags) {
 					resources = append(resources, &resource)
 				} else {
-					iface.logger.Debug("Skipping resource because search tags do not match")
+					iface.logger.Debug("Skipping resource because search tags do not match", "arn", resource.ARN)
 				}
 			}
 			return !lastPage


Test patch:
diff --git a/pkg/aws_cloudwatch_test.go b/pkg/aws_cloudwatch_test.go
index 648ab5d1c..f42010000 100644
--- a/pkg/aws_cloudwatch_test.go
+++ b/pkg/aws_cloudwatch_test.go
@@ -79,6 +79,89 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 		args               args
 		wantGetMetricsData []cloudwatchData
 	}{
+		{
+			"dimensionwrongname",
+			args{
+				region:     "us-east-1",
+				accountId:  aws.String("123123123123"),
+				namespace:  "ec2",
+				customTags: nil,
+				tagsOnMetrics: map[string][]string{
+					"ec2": {
+						"Value1",
+						"Value2",
+					},
+				},
+				dimensionRegexps: SupportedServices.GetService("ec2").DimensionRegexps,
+				resources: []*taggedResource{
+					{
+						ARN: "arn:aws:ec2:us-east-1:123123123123:instance/i-12312312312312312",
+						Tags: []Tag{
+							{
+								Key:   "Name",
+								Value: "some-Node",
+							},
+						},
+						Namespace: "ec2",
+						Region:    "us-east-1",
+					},
+				},
+				metricsList: []*cloudwatch.Metric{
+					{
+						MetricName: aws.String("CPUUtilization"),
+						Dimensions: []*cloudwatch.Dimension{
+							{
+								Name:  aws.String("BadDimension"),
+								Value: aws.String("lol"),
+							},
+						},
+						Namespace: aws.String("AWS/EC2"),
+					},
+				},
+				m: &Metric{
+					Name: "CPUUtilization",
+					Statistics: []string{
+						"Average",
+					},
+					Period:                 60,
+					Length:                 600,
+					Delay:                  120,
+					NilToZero:              aws.Bool(false),
+					AddCloudwatchTimestamp: aws.Bool(false),
+				},
+			},
+			[]cloudwatchData{
+				{
+					AccountId:              aws.String("123123123123"),
+					AddCloudwatchTimestamp: aws.Bool(false),
+					Dimensions: []*cloudwatch.Dimension{
+						{
+							Name:  aws.String("InstanceId"),
+							Value: aws.String("i-12312312312312312"),
+						},
+					},
+					ID:        aws.String("arn:aws:ec2:us-east-1:123123123123:instance/i-12312312312312312"),
+					Metric:    aws.String("CPUUtilization"),
+					Namespace: aws.String("ec2"),
+					NilToZero: aws.Bool(false),
+					Period:    60,
+					Region:    aws.String("us-east-1"),
+					Statistics: []string{
+						"Average",
+					},
+					Tags: []Tag{
+						{
+							Key:   "Value1",
+							Value: "",
+						},
+						{
+							Key:   "Value2",
+							Value: "",
+						},
+					},
+				},
+			},
+		},
 		{
 			"ec2",
 			args{