Repository: getmoto/moto

Issue:
Cloudwatch Moto implementation does not support Expressions
## Reporting Bug
**Moto version: 1.3.16**
**Python version: 3.8**
**Boto3 version: 1.14.63**

Installed this in my venv with `pip install moto`

## How to reproduce
Use the cloudwatch client with `get_metric_data` with an expression.
```
def test_function():
 result = cloudwatch.get_metric_data(
        MetricDataQueries=[
            {
                'Id': 'expression',
                'Expression': '(totalBytes/1048576)/PERIOD(totalBytes)*100,
                'Label': 'e1',
                'ReturnData': True,
            },
            {
                'Id': 'totalBytes',
                'MetricStat': {
                    'Metric': {
                        'Namespace': 'AWS/EFS',
                        'MetricName': 'TotalIOBytes',
                        'Dimensions': [
                            {
                                'Name': 'FileSystemId',
                                'Value': 'somestring'
                            },
                        ]
                    },
                    'Period': 60,
                    'Stat': 'Sum',
                    'Unit': 'Bytes',
                },
                'ReturnData': False,
            },
          ],
        StartTime=current_datetime - timedelta(minutes=5),
        EndTime=current_datetime,
        ScanBy='TimestampDescending',
        MaxDatapoints=123
      )
```
An example stacktrace when testing this with Pytest and a test setup looking something like this:
```
import boto3
from moto import mock_cloudwatch
import pytest

@pytest.fixture(scope='function')
def aws_credentials():
    """Mocked AWS Credentials for moto."""
    os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
    os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
    os.environ['AWS_SECURITY_TOKEN'] = 'testing'
    os.environ['AWS_SESSION_TOKEN'] = 'testing'

@pytest.fixture(scope='function')
def cloudwatch(aws_credentials):
    with mock_cloudwatch():
        yield boto3.client('cloudwatch', region_name='eu-west-1')

@mock_cloudwatch
def test__something():
from cloudwatch_example import test_function

boto3.client('cloudwatch', region_name='eu-west-1')
result = test_function()

```

This results in something like this:
```
self = <moto.cloudwatch.models.CloudWatchBackend object at 0x000001ACF01AED00>
queries = [{'expression': '(totalBytes/1048576)/PERIOD(totalBytes)*100/(throughput/1048576)', 'id': 'calculatedThroughput', 'lab..._stat._metric._dimensions.member.1._value': 'test_id', 'metric_stat._metric._metric_name': 'PermittedThroughput', ...}]
start_time = datetime.datetime(2020, 9, 18, 13, 32, 49, 853054, tzinfo=tzutc()), end_time = datetime.datetime(2020, 9, 18, 13, 37, 49, 853054, tzinfo=tzutc())     

    def get_metric_data(self, queries, start_time, end_time):
        period_data = [
            md for md in self.metric_data if start_time <= md.timestamp <= end_time
        ]
        results = []
        for query in queries:
>           query_ns = query["metric_stat._metric._namespace"]
E           KeyError: 'metric_stat._metric._namespace'

venv\lib\site-packages\moto\cloudwatch\models.py:339: KeyError
```

## Expected
I expected that it wouldn't return an error, because this key is not required when trying to query an expression. In other words, I expected a `MetricDataResults` dictionary returned.
Refering to: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudwatch.html#CloudWatch.Client.get_metric_data
It seems like the implementation for the `CloudWatchBackend` always requires a `MetricStat` with a `Metric` including a `Namespace`.





Discussion:
Correct @Stukongeluk - the `get_metric_data`-implementation is quite bare-bone at the moment. Marking this as an enhancement.

Thanks for raising it!

Resolving patch:
diff --git a/moto/cloudwatch/metric_data_expression_parser.py b/moto/cloudwatch/metric_data_expression_parser.py
new file mode 100644
--- /dev/null
+++ b/moto/cloudwatch/metric_data_expression_parser.py
@@ -0,0 +1,13 @@
+from typing import Any, Dict, List, Tuple, SupportsFloat
+
+
+def parse_expression(
+    expression: str, results: List[Dict[str, Any]]
+) -> Tuple[List[SupportsFloat], List[str]]:
+    values: List[SupportsFloat] = []
+    timestamps: List[str] = []
+    for result in results:
+        if result.get("id") == expression:
+            values.extend(result["vals"])
+            timestamps.extend(result["timestamps"])
+    return values, timestamps
diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py
--- a/moto/cloudwatch/models.py
+++ b/moto/cloudwatch/models.py
@@ -17,6 +17,7 @@
     ResourceNotFoundException,
     InvalidParameterCombination,
 )
+from .metric_data_expression_parser import parse_expression
 from .utils import make_arn_for_dashboard, make_arn_for_alarm
 from dateutil import parser
 from typing import Tuple, Optional, List, Iterable, Dict, Any, SupportsFloat
@@ -668,16 +669,23 @@ def get_metric_data(
         ]
 
         results = []
-        for query in queries:
+        results_to_return = []
+        metric_stat_queries = [q for q in queries if "MetricStat" in q]
+        expression_queries = [q for q in queries if "Expression" in q]
+        for query in metric_stat_queries:
             period_start_time = start_time
-            query_ns = query["metric_stat._metric._namespace"]
-            query_name = query["metric_stat._metric._metric_name"]
-            delta = timedelta(seconds=int(query["metric_stat._period"]))
-            dimensions = self._extract_dimensions_from_get_metric_data_query(query)
-            unit = query.get("metric_stat._unit")
+            metric_stat = query["MetricStat"]
+            query_ns = metric_stat["Metric"]["Namespace"]
+            query_name = metric_stat["Metric"]["MetricName"]
+            delta = timedelta(seconds=int(metric_stat["Period"]))
+            dimensions = [
+                Dimension(name=d["Name"], value=d["Value"])
+                for d in metric_stat["Metric"].get("Dimensions", [])
+            ]
+            unit = metric_stat.get("Unit")
             result_vals: List[SupportsFloat] = []
             timestamps: List[str] = []
-            stat = query["metric_stat._stat"]
+            stat = metric_stat["Stat"]
             while period_start_time <= end_time:
                 period_end_time = period_start_time + delta
                 period_md = [
@@ -715,21 +723,37 @@ def get_metric_data(
                 timestamps.reverse()
                 result_vals.reverse()
 
-            label = (
-                query["label"]
-                if "label" in query
-                else query["metric_stat._metric._metric_name"] + " " + stat
-            )
+            label = query.get("Label") or f"{query_name} {stat}"
 
             results.append(
                 {
-                    "id": query["id"],
+                    "id": query["Id"],
+                    "label": label,
+                    "vals": result_vals,
+                    "timestamps": timestamps,
+                }
+            )
+            if query.get("ReturnData", "true") == "true":
+                results_to_return.append(
+                    {
+                        "id": query["Id"],
+                        "label": label,
+                        "vals": result_vals,
+                        "timestamps": timestamps,
+                    }
+                )
+        for query in expression_queries:
+            label = query.get("Label") or f"{query_name} {stat}"
+            result_vals, timestamps = parse_expression(query["Expression"], results)
+            results_to_return.append(
+                {
+                    "id": query["Id"],
                     "label": label,
                     "vals": result_vals,
                     "timestamps": timestamps,
                 }
             )
-        return results
+        return results_to_return
 
     def get_metric_statistics(
         self,
@@ -904,23 +928,6 @@ def _get_paginated(
         else:
             return None, metrics
 
-    def _extract_dimensions_from_get_metric_data_query(
-        self, query: Dict[str, str]
-    ) -> List[Dimension]:
-        dimensions = []
-        prefix = "metric_stat._metric._dimensions.member."
-        suffix_name = "._name"
-        suffix_value = "._value"
-        counter = 1
-
-        while query.get(f"{prefix}{counter}{suffix_name}") and counter <= 10:
-            name = query.get(f"{prefix}{counter}{suffix_name}")
-            value = query.get(f"{prefix}{counter}{suffix_value}")
-            dimensions.append(Dimension(name=name, value=value))
-            counter = counter + 1
-
-        return dimensions
-
     def _validate_parameters_put_metric_data(
         self, metric: Dict[str, Any], query_num: int
     ) -> None:
diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py
--- a/moto/cloudwatch/responses.py
+++ b/moto/cloudwatch/responses.py
@@ -13,7 +13,7 @@
     Dimension,
     FakeAlarm,
 )
-from .exceptions import InvalidParameterCombination
+from .exceptions import InvalidParameterCombination, ValidationError
 
 
 ERROR_RESPONSE = Tuple[str, Dict[str, int]]
@@ -176,11 +176,18 @@ def put_metric_data(self) -> str:
 
     @amzn_request_id
     def get_metric_data(self) -> str:
-        start = dtparse(self._get_param("StartTime"))
-        end = dtparse(self._get_param("EndTime"))
-        scan_by = self._get_param("ScanBy")
-
-        queries = self._get_list_prefix("MetricDataQueries.member")
+        params = self._get_params()
+        start = dtparse(params["StartTime"])
+        end = dtparse(params["EndTime"])
+        scan_by = params.get("ScanBy") or "TimestampDescending"
+
+        queries = params.get("MetricDataQueries", [])
+        for query in queries:
+            if "MetricStat" not in query and "Expression" not in query:
+                # AWS also returns the empty line
+                raise ValidationError(
+                    "The parameter MetricDataQueries.member.1.MetricStat is required.\n"
+                )
         results = self.cloudwatch_backend.get_metric_data(
             start_time=start, end_time=end, queries=queries, scan_by=scan_by
         )


Test patch:
diff --git a/tests/test_cloudwatch/test_cloudwatch_expression_parser.py b/tests/test_cloudwatch/test_cloudwatch_expression_parser.py
new file mode 100644
--- /dev/null
+++ b/tests/test_cloudwatch/test_cloudwatch_expression_parser.py
@@ -0,0 +1,40 @@
+from moto.cloudwatch.metric_data_expression_parser import parse_expression
+
+
+def test_simple_expression():
+    result_from_previous_queries = [
+        {
+            "id": "totalBytes",
+            "label": "metric Sum",
+            "vals": [25.0],
+            "timestamps": ["timestamp1"],
+        }
+    ]
+    res = parse_expression("totalBytes", result_from_previous_queries)
+    assert res == ([25.0], ["timestamp1"])
+
+
+def test_missing_expression():
+    result_from_previous_queries = [
+        {
+            "id": "totalBytes",
+            "label": "metric Sum",
+            "vals": [25.0],
+            "timestamps": ["timestamp1"],
+        }
+    ]
+    res = parse_expression("unknown", result_from_previous_queries)
+    assert res == ([], [])
+
+
+def test_complex_expression():
+    result_from_previous_queries = [
+        {
+            "id": "totalBytes",
+            "label": "metric Sum",
+            "vals": [25.0],
+            "timestamps": ["timestamp1"],
+        }
+    ]
+    res = parse_expression("totalBytes/10", result_from_previous_queries)
+    assert res == ([], [])
diff --git a/tests/test_cloudwatch/test_cloudwatch_expressions.py b/tests/test_cloudwatch/test_cloudwatch_expressions.py
new file mode 100644
--- /dev/null
+++ b/tests/test_cloudwatch/test_cloudwatch_expressions.py
@@ -0,0 +1,70 @@
+import boto3
+import pytest
+
+from botocore.exceptions import ClientError
+from datetime import datetime, timedelta, timezone
+from moto import mock_cloudwatch
+
+
+@mock_cloudwatch
+def test_get_metric_data__no_metric_data_or_expression():
+    utc_now = datetime.now(tz=timezone.utc)
+    cloudwatch = boto3.client("cloudwatch", "eu-west-1")
+    with pytest.raises(ClientError) as exc:
+        cloudwatch.get_metric_data(
+            MetricDataQueries=[{"Id": "result1", "Label": "e1"}],
+            StartTime=utc_now - timedelta(minutes=5),
+            EndTime=utc_now,
+            ScanBy="TimestampDescending",
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationError"
+    assert (
+        err["Message"]
+        == "The parameter MetricDataQueries.member.1.MetricStat is required.\n"
+    )
+
+
+@mock_cloudwatch
+def test_get_metric_data_with_simple_expression():
+    utc_now = datetime.now(tz=timezone.utc)
+    cloudwatch = boto3.client("cloudwatch", "eu-west-1")
+    namespace = "my_namespace/"
+    cloudwatch.put_metric_data(
+        Namespace=namespace,
+        MetricData=[
+            {
+                "MetricName": "metric",
+                "Value": 25,
+                "Unit": "Bytes",
+            },
+        ],
+    )
+    # get_metric_data 1
+    results = cloudwatch.get_metric_data(
+        MetricDataQueries=[
+            {
+                "Id": "result1",
+                "Expression": "totalBytes",
+                "Label": "e1",
+            },
+            {
+                "Id": "totalBytes",
+                "MetricStat": {
+                    "Metric": {"Namespace": namespace, "MetricName": "metric"},
+                    "Period": 60,
+                    "Stat": "Sum",
+                    "Unit": "Bytes",
+                },
+                "ReturnData": False,
+            },
+        ],
+        StartTime=utc_now - timedelta(minutes=5),
+        EndTime=utc_now + timedelta(minutes=5),
+        ScanBy="TimestampDescending",
+    )["MetricDataResults"]
+    #
+    assert len(results) == 1
+    assert results[0]["Id"] == "result1"
+    assert results[0]["Label"] == "e1"
+    assert results[0]["Values"] == [25.0]