Repository: Sceptre/sceptre-cdk-handler (language: python)

Issue:
Main CDK module is not able to import other local modules
The main sceptre CDK module is not able to import any other modules.  I'm getting a `ModuleNotFoundError` when attempting to do the import.   Tried importing using both relative and non-relative notations and both fail.  My entire sceptre test project can be downloaded from https://file.io/J0yCGEMdQEdx

My project:

➜  testprojB tree

```
.
├── config
│   ├── config.yaml
│   └── develop
│       ├── app.yaml
│       └── config.yaml
├── package-lock.json
├── package.json
├── requirements-dev.txt
├── requirements.txt
└── templates
    ├── __init__.py
    ├── __pycache__
    │   └── app.cpython-39.pyc
    ├── app.py
    └── my_project
        ├── __init__.py
        └── my_project_stack.py

```

➜  testprojB cat templates/app.py
```
from aws_cdk import CfnParameter
from sceptre_cdk_handler import SceptreCdkStack
from my_project.my_project_stack import MyProjectStack

# Important: Notice how it subclasses SceptreCdkStack and passes **kwargs into the base class's
# __init__(). This is important to maintain compability with the different deployment_types.
class MyAppStack(SceptreCdkStack):
    def __init__(self, scope, id: str, sceptre_user_data: dict, **kwargs):
        super().__init__(scope, id, sceptre_user_data, **kwargs)
        # If you want to pass parameters like you do elsewhere in Sceptre, this works great!
        self.special_parameter = CfnParameter(self, 'SpecialEnv')
        self.my_project_stack = MyProjectStack(self)
```

Sceptre execution:
```
➜  testprojB AWS_PROFILE=itsandbox-admin AWS_DEFAULT_REGION=us-east-1 sceptre generate develop/app.yaml
testprojB@ /Users/zaro0508/work-sage/testprojB
└── cdk-assets@2.63.0

Traceback (most recent call last):
  File "/Users/zaro0508/.pyenv/versions/py39-data_curator-infra/bin/sceptre", line 8, in <module>
    sys.exit(cli())
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1130, in __call__
    return self.main(*args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1055, in main
    rv = self.invoke(ctx)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1657, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 760, in invoke
    return __callback(*args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/decorators.py", line 26, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/cli/helpers.py", line 42, in decorated
    return func(*args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/cli/template.py", line 89, in generate_command
    responses = plan.generate()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/plan.py", line 387, in generate
    return self._execute(*args)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/plan.py", line 29, in wrapped
    return func(self, *args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/plan.py", line 56, in _execute
    return executor.execute(*args)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/executor.py", line 52, in execute
    stack, status = future.result()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/lib/python3.9/concurrent/futures/_base.py", line 438, in result
    return self.__get_result()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result
    raise self._exception
  File "/Users/zaro0508/.pyenv/versions/3.9.7/lib/python3.9/concurrent/futures/thread.py", line 52, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/executor.py", line 59, in _execute
    result = getattr(actions, self.command)(*args)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/hooks/__init__.py", line 104, in decorated
    response = func(self, *args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/actions.py", line 641, in generate
    return self.stack.template.body
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/template.py", line 90, in body
    body = handler.handle()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre_cdk_handler/cdk.py", line 157, in handle
    stack_class: Type[SceptreCdkStack] = self._importer.import_class(self.cdk_template_path, self.cdk_class_name)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre_cdk_handler/class_importer.py", line 31, in import_class
    loader.exec_module(template_module)
  File "<frozen importlib._bootstrap_external>", line 850, in exec_module
  File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removed
  File "/Users/zaro0508/work-sage/testprojB/templates/app.py", line 3, in <module>
    from my_project.my_project_stack import MyProjectStack
ModuleNotFoundError: No module named 'my_project'
```


Main CDK module is not able to import other local modules
The main sceptre CDK module is not able to import any other modules.  I'm getting a `ModuleNotFoundError` when attempting to do the import.   Tried importing using both relative and non-relative notations and both fail.  My entire sceptre test project can be downloaded from https://file.io/J0yCGEMdQEdx

My project:

➜  testprojB tree

```
.
├── config
│   ├── config.yaml
│   └── develop
│       ├── app.yaml
│       └── config.yaml
├── package-lock.json
├── package.json
├── requirements-dev.txt
├── requirements.txt
└── templates
    ├── __init__.py
    ├── __pycache__
    │   └── app.cpython-39.pyc
    ├── app.py
    └── my_project
        ├── __init__.py
        └── my_project_stack.py

```

➜  testprojB cat templates/app.py
```
from aws_cdk import CfnParameter
from sceptre_cdk_handler import SceptreCdkStack
from my_project.my_project_stack import MyProjectStack

# Important: Notice how it subclasses SceptreCdkStack and passes **kwargs into the base class's
# __init__(). This is important to maintain compability with the different deployment_types.
class MyAppStack(SceptreCdkStack):
    def __init__(self, scope, id: str, sceptre_user_data: dict, **kwargs):
        super().__init__(scope, id, sceptre_user_data, **kwargs)
        # If you want to pass parameters like you do elsewhere in Sceptre, this works great!
        self.special_parameter = CfnParameter(self, 'SpecialEnv')
        self.my_project_stack = MyProjectStack(self)
```

Sceptre execution:
```
➜  testprojB AWS_PROFILE=itsandbox-admin AWS_DEFAULT_REGION=us-east-1 sceptre generate develop/app.yaml
testprojB@ /Users/zaro0508/work-sage/testprojB
└── cdk-assets@2.63.0

Traceback (most recent call last):
  File "/Users/zaro0508/.pyenv/versions/py39-data_curator-infra/bin/sceptre", line 8, in <module>
    sys.exit(cli())
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1130, in __call__
    return self.main(*args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1055, in main
    rv = self.invoke(ctx)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1657, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/core.py", line 760, in invoke
    return __callback(*args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/click/decorators.py", line 26, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/cli/helpers.py", line 42, in decorated
    return func(*args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/cli/template.py", line 89, in generate_command
    responses = plan.generate()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/plan.py", line 387, in generate
    return self._execute(*args)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/plan.py", line 29, in wrapped
    return func(self, *args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/plan.py", line 56, in _execute
    return executor.execute(*args)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/executor.py", line 52, in execute
    stack, status = future.result()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/lib/python3.9/concurrent/futures/_base.py", line 438, in result
    return self.__get_result()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result
    raise self._exception
  File "/Users/zaro0508/.pyenv/versions/3.9.7/lib/python3.9/concurrent/futures/thread.py", line 52, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/executor.py", line 59, in _execute
    result = getattr(actions, self.command)(*args)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/hooks/__init__.py", line 104, in decorated
    response = func(self, *args, **kwargs)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/plan/actions.py", line 641, in generate
    return self.stack.template.body
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre/template.py", line 90, in body
    body = handler.handle()
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre_cdk_handler/cdk.py", line 157, in handle
    stack_class: Type[SceptreCdkStack] = self._importer.import_class(self.cdk_template_path, self.cdk_class_name)
  File "/Users/zaro0508/.pyenv/versions/3.9.7/envs/py39-data_curator-infra/lib/python3.9/site-packages/sceptre_cdk_handler/class_importer.py", line 31, in import_class
    loader.exec_module(template_module)
  File "<frozen importlib._bootstrap_external>", line 850, in exec_module
  File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removed
  File "/Users/zaro0508/work-sage/testprojB/templates/app.py", line 3, in <module>
    from my_project.my_project_stack import MyProjectStack
ModuleNotFoundError: No module named 'my_project'
```




Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 431b455..fb2a7ea 100644
--- a/README.md
+++ b/README.md
@@ -272,6 +272,53 @@ In other words, using this handler lets you use resolvers, put your CDK stack in
 you name your stack according to Sceptre's naming conventions, `validate`, `diff`, and more! Basically,
 the CDK stack can be managed using Sceptre just like any other.
 
+### Importing from other files
+Your CDK Stack module _can_ import from other packages/modules in your local directory structure.
+This is useful to break your constructs up with a file-based organization. While you could
+theoretically point the CDK handler to any python file path on your computer, there are limitations
+on how other modules will be accessible for import:
+
+1. All modules/packages to be imported **must be somewhere inside your current working directory.**
+If the files you intend to import are _outside_ your CWD, they will not be accessible.
+2. They must _either_...
+   1. Be in the immediate directory structure between your CWD and the python file your
+handler is referencing via `path` _or_
+   2. Be inside importable python packages that _are_ in the immediate structure between your CWD and
+your handler's `path`.
+
+```
+other_directory/
+    file.py                         # You CANNOT import this because it is out of your CWD
+project_directory/
+   templates/                       # No __init__.py in here, but it's in the directory hierarchy of
+                                    # your_cdk_stack.py, so you can technically import any direct
+                                    # children of this directory.
+
+       random_python_file.py        # You CAN import this ("import random_python_file")
+       unrelated_directory/         # No __init__.py in here...
+           nested_dir/              # No __init__.py in here...
+               some_python_file.py  # You CANNOT import this because this isn't in a python package
+       my_cdk_files/
+           __init__.py              # this lets you import from this dir (my_cdk_files/)
+           constructs/
+               __init__.py          # This lets you import from this dir (constructs/)
+               ec2/
+                   __init__.py      # This lets you import from this dir (ec2/)
+                   vpc.py           # You CAN import this (my_cdk_files.constructs.ec2.vpc)
+                   autoscaling.py   # You CAN import this (my_cdk_files.constructs.ec2.autoscaling)
+           stacks/
+               __init__.py          # This lets you import from this dir (my_cdk_files/)
+               >> your_cdk_stack.py # This is where your CDK Handler points to
+   scripts/
+      some_dir/
+          random_python_file.py     # You cannot import this because it's not in the directory
+                                    # hierarchy of your_cdk_stack.py and it's not in a python package
+                                    # importable from any of the directories that ARE in that hierarchy.
+   app/
+       __init__.py                  # This makes app/ an importable python package
+       my_application_resources.py  # You CAN import this (app.my_application_resources)
+```
+
 ### IAM and authentication
 
 There are several dimensions to how using this handler applies to IAM roles, policies, and permissions.
diff --git a/sceptre_cdk_handler/class_importer.py b/sceptre_cdk_handler/class_importer.py
index 98963a6..c0db685 100644
--- a/sceptre_cdk_handler/class_importer.py
+++ b/sceptre_cdk_handler/class_importer.py
@@ -1,5 +1,6 @@
 import importlib.machinery
 import importlib.util
+import sys
 from pathlib import Path
 from typing import Type
 
@@ -24,7 +25,8 @@ def import_class(self, template_path: Path, class_name: str) -> Type[SceptreCdkS
             SceptreException: Template File not found
             SceptreException: importlib general exception
         """
-        template_module_name = template_path.stem
+        template_module_name = self._enable_import_hierarchy(template_path)
+
         loader = importlib.machinery.SourceFileLoader(template_module_name, str(template_path))
         spec = importlib.util.spec_from_loader(template_module_name, loader)
         template_module = importlib.util.module_from_spec(spec)
@@ -36,3 +38,38 @@ def import_class(self, template_path: Path, class_name: str) -> Type[SceptreCdkS
             raise exceptions.SceptreException(
                 f"No class named  {class_name} on template at {template_path}"
             )
+
+    def _enable_import_hierarchy(self, template_path: Path) -> str:
+        resolved_template_path = template_path.resolve()
+        cwd = Path.cwd()
+        # If the template path we're importing isn't somewhere in the CWD, we can't know how far up
+        # to go with adding directories to the PATH, so we're not going to try. That could get kinda
+        # screwy and cause unintended consequences.
+        if cwd not in resolved_template_path.parents:
+            # We'll consider the file name (without the stem) to be the module name.
+            return template_path.stem
+
+        module_path_segments = [template_path.stem]
+        in_package_structure = True
+        # We're going to climb up the hierarchy and add the whole directory structure to the PATH.
+        # This would theoretically allow for imports from any level of the hierarchy. It's not ideal
+        # but it's really the only way we can know how high up the import chain goes. However, we do
+        # require each directory to have an __init__.py to consider it a part of the importable
+        # hierarchy.
+        for parent in resolved_template_path.parents:
+            sys.path.append(str(parent))
+            # If the parent directory is a valid python package in name and structure, we'll add it
+            # to the module name segments and keep climbing
+            if in_package_structure and (parent / '__init__.py').exists() and parent.name.isidentifier():
+                module_path_segments.insert(0, parent.name)
+            # But if the parent directory isn't a valid python package in name and structure, we'll
+            # stop building out the module path.
+            elif in_package_structure:
+                in_package_structure = False
+
+            # If we've climbed all the way up to the CWD.
+            if parent == cwd:
+                break
+
+        # We'll make the full module path by joining all the segments together.
+        return '.'.join(module_path_segments)


Test patch:
diff --git a/tests/assets/__init__.py b/tests/assets/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/assets/parent_dir/__init__.py b/tests/assets/parent_dir/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/assets/parent_dir/inner_dir/__init__.py b/tests/assets/parent_dir/inner_dir/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/assets/parent_dir/inner_dir/class_dir/__init__.py b/tests/assets/parent_dir/inner_dir/class_dir/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/assets/parent_dir/inner_dir/class_dir/file_to_import.py b/tests/assets/parent_dir/inner_dir/class_dir/file_to_import.py
new file mode 100644
index 0000000..d90fcb2
--- /dev/null
+++ b/tests/assets/parent_dir/inner_dir/class_dir/file_to_import.py
@@ -0,0 +1,8 @@
+from parent_dir.inner_dir.class_dir import file_to_import_with_full_path
+from . import file_to_import_with_relative_import
+
+
+class MyFancyClassToImport:
+    full_path_import = file_to_import_with_full_path.ATTRIBUTE_VALUE
+    relative_path_import = file_to_import_with_relative_import.ATTRIBUTE_VALUE
+    attribute = 'Success!'
diff --git a/tests/assets/parent_dir/inner_dir/class_dir/file_to_import_with_full_path.py b/tests/assets/parent_dir/inner_dir/class_dir/file_to_import_with_full_path.py
new file mode 100644
index 0000000..ba4c358
--- /dev/null
+++ b/tests/assets/parent_dir/inner_dir/class_dir/file_to_import_with_full_path.py
@@ -0,0 +1,1 @@
+ATTRIBUTE_VALUE = "Success!"
diff --git a/tests/assets/parent_dir/inner_dir/class_dir/file_to_import_with_relative_import.py b/tests/assets/parent_dir/inner_dir/class_dir/file_to_import_with_relative_import.py
new file mode 100644
index 0000000..ba4c358
--- /dev/null
+++ b/tests/assets/parent_dir/inner_dir/class_dir/file_to_import_with_relative_import.py
@@ -0,0 +1,1 @@
+ATTRIBUTE_VALUE = "Success!"
diff --git a/tests/test_class_importer.py b/tests/test_class_importer.py
index 6799f53..cfaee5a 100644
--- a/tests/test_class_importer.py
+++ b/tests/test_class_importer.py
@@ -8,17 +8,16 @@
 
 class TestClassImporter(TestCase):
     def test_import_class__imports_class_from_referenced_file(self):
-        filepath = Path(__file__).parent / 'assets' / 'file_to_import.py'
+        filepath = Path(__file__).parent / 'assets' / 'parent_dir' / 'inner_dir' / 'class_dir' / 'file_to_import.py'
         class_name = "MyFancyClassToImport"
         importer = ClassImporter()
         result = importer.import_class(filepath, class_name)
-        self.assertEqual(
-            'Success!',
-            result.attribute
-        )
+        self.assertEqual('Success!', result.attribute)
+        self.assertEqual('Success!', result.full_path_import)
+        self.assertEqual('Success!', result.relative_path_import)
 
     def test_import_class__named_class_isnt_on_module__raises_sceptre_exception(self):
-        filepath = Path(__file__).parent / 'assets' / 'file_to_import.py'
+        filepath = Path(__file__).parent / 'assets' / 'parent_dir' / 'inner_dir' / 'class_dir' / 'file_to_import.py'
         importer = ClassImporter()
         with self.assertRaises(SceptreException):
             importer.import_class(filepath, "CantFindMe")