Repository: linz/geostore

Issue:
Infra: split "datalake" stack to storage and endpoints stacks
Split "datalake" stack to storage and endpoints stacks


Discussion:

Resolving patch:
diff --git a/infra/app.py b/infra/app.py
--- a/infra/app.py
+++ b/infra/app.py
@@ -7,9 +7,10 @@
 import os
 
 from aws_cdk import core
-from datalake.datalake_stack import DataLakeStack  # pylint: disable=import-error
+from datalake.api_stack import APIStack  # pylint: disable=import-error
 from datalake.networking_stack import NetworkingStack  # pylint: disable=import-error
 from datalake.processing_stack import ProcessingStack  # pylint: disable=import-error
+from datalake.storage_stack import StorageStack  # pylint: disable=import-error
 
 ENV = os.environ.get("DEPLOY_ENV", "dev")
 
@@ -23,6 +24,14 @@
     env={"region": os.environ["CDK_DEFAULT_REGION"], "account": os.environ["CDK_DEFAULT_ACCOUNT"]},
 )
 
+storage = StorageStack(
+    app,
+    "storage",
+    stack_name=f"geospatial-data-lake-storage-{ENV}",
+    env={"region": os.environ["CDK_DEFAULT_REGION"], "account": os.environ["CDK_DEFAULT_ACCOUNT"]},
+    deploy_env=ENV,
+)
+
 processing = ProcessingStack(
     app,
     "processing",
@@ -32,12 +41,12 @@
     vpc=networking.datalake_vpc,
 )
 
-datalake = DataLakeStack(
+api = APIStack(
     app,
-    "datalake",
-    stack_name=f"geospatial-data-lake-{ENV}",
+    "api",
+    stack_name=f"geospatial-data-lake-api-{ENV}",
     env={"region": os.environ["CDK_DEFAULT_REGION"], "account": os.environ["CDK_DEFAULT_ACCOUNT"]},
-    deploy_env=ENV,
+    datasets_table=storage.datasets_table,
 )
 
 # tag all resources in stack
diff --git a/infra/datalake/api_stack.py b/infra/datalake/api_stack.py
new file mode 100644
--- /dev/null
+++ b/infra/datalake/api_stack.py
@@ -0,0 +1,42 @@
+"""
+Data Lake AWS resources definitions.
+"""
+from aws_cdk import aws_lambda, core
+from aws_cdk.core import Tags
+
+
+class APIStack(core.Stack):
+    """Data Lake stack definition."""
+
+    def __init__(self, scope: core.Construct, stack_id: str, datasets_table, **kwargs) -> None:
+        super().__init__(scope, stack_id, **kwargs)
+
+        ############################################################################################
+        # ### API ENDPOINTS ########################################################################
+        ############################################################################################
+
+        endpoints = ("datasets",)
+
+        for endpoint in endpoints:
+            endpoint_function = aws_lambda.Function(
+                self,
+                f"{endpoint}-endpoint-function",
+                function_name=f"{endpoint}-endpoint",
+                handler=f"endpoints.{endpoint}.entrypoint.lambda_handler",
+                runtime=aws_lambda.Runtime.PYTHON_3_6,
+                code=aws_lambda.Code.from_asset(
+                    path="..",
+                    bundling=core.BundlingOptions(
+                        # pylint:disable=no-member
+                        image=aws_lambda.Runtime.PYTHON_3_6.bundling_docker_image,
+                        command=["backend/bundle.bash", f"endpoints/{endpoint}"],
+                    ),
+                ),
+            )
+
+            datasets_table.grant_read_write_data(endpoint_function)
+            datasets_table.grant(
+                endpoint_function, "dynamodb:DescribeTable"
+            )  # required by pynamodb
+
+            Tags.of(endpoint_function).add("ApplicationLayer", "api")
diff --git a/infra/datalake/datalake_stack.py b/infra/datalake/storage_stack.py
similarity index 56%
rename from infra/datalake/datalake_stack.py
rename to infra/datalake/storage_stack.py
--- a/infra/datalake/datalake_stack.py
+++ b/infra/datalake/storage_stack.py
@@ -1,13 +1,11 @@
 """
 Data Lake AWS resources definitions.
 """
-from aws_cdk import aws_dynamodb, aws_lambda, aws_s3, core
+from aws_cdk import aws_dynamodb, aws_s3, core
 from aws_cdk.core import Tags
 
 
-class DataLakeStack(core.Stack):
-    """Data Lake stack definition."""
-
+class StorageStack(core.Stack):
     def __init__(self, scope: core.Construct, stack_id: str, deploy_env, **kwargs) -> None:
         super().__init__(scope, stack_id, **kwargs)
 
@@ -20,7 +18,7 @@ def __init__(self, scope: core.Construct, stack_id: str, deploy_env, **kwargs) -
         ############################################################################################
         # ### STORAGE S3 BUCKET ####################################################################
         ############################################################################################
-        storage_bucket = aws_s3.Bucket(
+        self.storage_bucket = aws_s3.Bucket(
             self,
             "storage-bucket",
             bucket_name="{}-{}".format(
@@ -31,12 +29,12 @@ def __init__(self, scope: core.Construct, stack_id: str, deploy_env, **kwargs) -
             versioned=True,
             removal_policy=REMOVAL_POLICY,
         )
-        Tags.of(storage_bucket).add("ApplicationLayer", "storage")
+        Tags.of(self.storage_bucket).add("ApplicationLayer", "storage")
 
         ############################################################################################
         # ### APPLICATION DB #######################################################################
         ############################################################################################
-        app_db_datasets = aws_dynamodb.Table(
+        self.datasets_table = aws_dynamodb.Table(
             self,
             "application-db",
             table_name="datasets",
@@ -46,12 +44,12 @@ def __init__(self, scope: core.Construct, stack_id: str, deploy_env, **kwargs) -
             removal_policy=REMOVAL_POLICY,
         )
 
-        app_db_datasets.add_global_secondary_index(
+        self.datasets_table.add_global_secondary_index(
             index_name="datasets_title",
             partition_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING),
             sort_key=aws_dynamodb.Attribute(name="title", type=aws_dynamodb.AttributeType.STRING),
         )
-        app_db_datasets.add_global_secondary_index(
+        self.datasets_table.add_global_secondary_index(
             index_name="datasets_owning_group",
             partition_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING),
             sort_key=aws_dynamodb.Attribute(
@@ -59,34 +57,4 @@ def __init__(self, scope: core.Construct, stack_id: str, deploy_env, **kwargs) -
             ),
         )
 
-        Tags.of(app_db_datasets).add("ApplicationLayer", "application-db")
-
-        ############################################################################################
-        # ### API ENDPOINTS ########################################################################
-        ############################################################################################
-
-        endpoints = ("datasets",)
-
-        for endpoint in endpoints:
-            endpoint_function = aws_lambda.Function(
-                self,
-                f"{endpoint}-endpoint-function",
-                function_name=f"{endpoint}-endpoint",
-                handler=f"endpoints.{endpoint}.entrypoint.lambda_handler",
-                runtime=aws_lambda.Runtime.PYTHON_3_6,
-                code=aws_lambda.Code.from_asset(
-                    path="..",
-                    bundling=core.BundlingOptions(
-                        # pylint:disable=no-member
-                        image=aws_lambda.Runtime.PYTHON_3_6.bundling_docker_image,
-                        command=["backend/bundle.bash", f"endpoints/{endpoint}"],
-                    ),
-                ),
-            )
-
-            app_db_datasets.grant_read_write_data(endpoint_function)
-            app_db_datasets.grant(
-                endpoint_function, "dynamodb:DescribeTable"
-            )  # required by pynamodb
-
-            Tags.of(endpoint_function).add("ApplicationLayer", "api")
+        Tags.of(self.datasets_table).add("ApplicationLayer", "application-db")


Test patch:
diff --git a/infra/tests/test_stack.py b/infra/tests/test_stack.py
deleted file mode 100644
--- a/infra/tests/test_stack.py
+++ /dev/null
@@ -1,23 +0,0 @@
-"""
-Data Lake stack tests.
-"""
-
-import os
-
-import boto3
-from pytest import mark
-
-CF = boto3.client("cloudformation")
-
-
-ENV = os.environ.get("DEPLOY_ENV", "dev")
-
-
-@mark.infrastructure
-def test_stack_create_complete():
-    """Test if CloudFormation stack is successfully created."""
-    response = CF.describe_stacks(StackName=f"geospatial-data-lake-{ENV}")
-    assert response["Stacks"][0]["StackStatus"] in (
-        "CREATE_COMPLETE",
-        "UPDATE_COMPLETE",
-    )