Repository: level12/keg

Issue:
component structure
Splitting an app up into more self-contained "components" is a helpful strategy for managing large projects. Flask gives us blueprints, which are great for segmenting off blocks of routes/views. And, keg has a hook on the app for listing out paths for models. These are the two pieces that need to be "hooked up" during app init, but IMO a developer should not need to manually hook these pieces up for every component.

Our working idea is to make components flask extensions that are responsible for their own setup:
- `REGISTERED_COMPONENTS` setting in app config for easy feature-flagging
  - use full dotted paths, so we're not tied to the app package necessarily
- model and blueprint setup handled via a generic component class
  - should be configurable for multiple model modules, or multiple blueprints per component
- component-specific config defaults could be handled in that component class as well
- the keg app's only real job here would be to call init_app on any registered components, during `init_extensions`

Discussion:

Patch:
diff --git a/keg/__init__.py b/keg/__init__.py
index 22732b4..cf62dce 100644
--- a/keg/__init__.py
+++ b/keg/__init__.py
@@ -1,9 +1,20 @@
 from __future__ import absolute_import
 
-from flask import current_app
+from flask import current_app  # noqa: F401
 
-from keg.app import Keg
+from keg.app import Keg  # noqa: F401
+from keg.component import (  # noqa: F401
+    KegComponent,
+    KegModelComponent,
+    KegModelViewComponent,
+    KegViewComponent,
+)
 
-# silence linter
-current_app
-Keg
+__all__ = [
+    'Keg',
+    'KegComponent',
+    'KegModelComponent',
+    'KegModelViewComponent',
+    'KegViewComponent',
+    'current_app',
+]
diff --git a/keg/app.py b/keg/app.py
index b580805..d974bb8 100644
--- a/keg/app.py
+++ b/keg/app.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import importlib
 import warnings
 
 import flask
@@ -135,6 +136,12 @@ class Keg(flask.Flask):
     def init_extensions(self):
         self.init_db()
 
+        # Keg components are essentially flask extensions, so they need to be set up during the
+        # init process. Being extensions, it makes sense to trigger this here. But, they will best
+        # be set up at the end of the init process, because other app-specific extensions may be
+        # set up in `on_init_complete` for error handling/notifications and the like.
+        signals.init_complete.connect(self.init_registered_components, sender=self)
+
     def db_manager_cls(self):
         from keg.db import DatabaseManager
         return DatabaseManager
@@ -144,6 +151,16 @@ class Keg(flask.Flask):
             cls = self.db_manager_cls()
             self.db_manager = cls(self)
 
+    def init_registered_components(self, app):
+        # KEG_REGISTERED_COMPONENTS is presumed to be a set/list/iterable of dotted paths usable
+        # for import. At the top level of the imported path, there should be a `__component__`
+        # that takes the dotted path that was used for import (as an absolute parent for relative
+        # imports) and has an init_app. Ideally, based on KegComponent.
+        for comp_path in self.config.get('KEG_REGISTERED_COMPONENTS', set()):
+            comp_module = importlib.import_module(comp_path)
+            comp_object = getattr(comp_module, '__component__')
+            comp_object.init_app(self, parent_path=comp_path)
+
     def init_blueprints(self):
         # TODO: probably want to be selective about adding our blueprint
         self.register_blueprint(kegbp)
diff --git a/keg/component.py b/keg/component.py
new file mode 100644
index 0000000..13fef79
--- /dev/null
+++ b/keg/component.py
@@ -0,0 +1,121 @@
+import importlib
+
+import flask
+
+
+class KegComponent:
+    """ Keg components follow the paradigm of flask extensions, and provide some defaults for the
+    purpose of setting up model/view structure. Using components, a project may be broken down into
+    logical blocks, each having their own entities, blueprints, templates, tests, etc.
+
+    Setup involves:
+    - KEG_REGISTERED_COMPONENTS config setting: assumed to be an iterable of importable dotted paths
+    - `__component__`: at the top level of each dotted path, this attribute should point to an
+    instance of `KegComponent`. E.g. `__component__ = KegComponent('widgets')`
+
+    By default, components will load entities from `db_visit_modules` into metadata and register
+    any blueprints specified by `load_blueprints`.
+
+    Blueprints can be created with the helper methods `create_named_blueprint` or `create_blueprint`
+    in order to have a configured template folder relative to the blueprint path.
+
+    Use KegModelComponent, KegViewComponent, or KegModelViewComponent for some helpful defaults for
+    model/blueprint discovery.
+
+    db_visit_modules: an iterable of dotted paths (e.g. `.mycomponent.entities`,
+                      `app.component.extraentities`) where Keg can find the entities for this
+                      component to load them into the metadata.
+
+    .. note:: Normally this is not explicitly required but can be useful in cases where imports
+       won't reach that file.
+
+    .. note:: This can accept relative dotted paths (starts with `.`) and it will prepend the
+       component python package determined by Keg when instantiating the component. You can also
+       pass absolute dotted paths and no alterations will be performed.
+
+    load_blueprints: an iterable of tuples, each having a dotted path (e.g. `.mycomponent.views`,
+                     `app.component.extraviews`) and the blueprint attribute name to load and
+                     register on the app. E.g. `(('.views', 'component_bp'), )`
+
+    .. note:: This can accept relative dotted paths (starts with `.`) and it will prepend the
+       component python package determined by Keg when instantiating the component. You can also
+       pass absolute dotted paths and no alterations will be performed.
+
+    template_folder: string to be passed for template config to blueprints created via the component
+    """
+    db_visit_modules = tuple()
+    load_blueprints = tuple()
+    template_folder = 'templates'
+
+    def __init__(self, name, app=None, db_visit_modules=None, load_blueprints=None,
+                 template_folder=None, parent_path=None):
+        self.name = name
+        # Allow customization of the defaults in the constructor
+        self.db_visit_modules = db_visit_modules or self.db_visit_modules
+        self.load_blueprints = load_blueprints or self.load_blueprints
+        self.template_folder = template_folder or self.template_folder
+        if app:
+            # Not really intended to be used this way, but it fits the flask extension paradigm
+            # and could conceivably be set up in lieu of KEG_REGISTERED_COMPONENTS
+            self.init_app(app, parent_path=parent_path)
+
+    def init_app(self, app, parent_path=None):
+        # parent_path gets used as an absolute parent for the relative import paths of
+        # model/blueprints.
+        # E.g. if relative_dotted_path is `my_app.components.widget` and one of the relative import
+        # paths is `.model.entities`, the full import is `my_app.components.widget.model.entities`
+        self.init_config(app)
+        self.init_db(parent_path)
+        self.init_blueprints(app, parent_path)
+
+    def init_config(self, app):
+        # Components may define their own config defaults
+        pass
+
+    def init_db(self, parent_path):
+        # Intent is to import the listed modules, so their entities are registered in metadata
+        for dotted_path in self.db_visit_modules:
+            import_name = dotted_path
+            if import_name.startswith('.'):
+                import_name = f'{parent_path}{dotted_path}'
+            importlib.import_module(import_name)
+
+    def init_blueprints(self, app, parent_path):
+        # Register any blueprints that are listed by path in load_blueprints
+        for bp_path, bp_attr in self.load_blueprints:
+            import_name = bp_path
+            if import_name.startswith('.'):
+                import_name = f'{parent_path}{bp_path}'
+            mod_imported = importlib.import_module(import_name)
+            app.register_blueprint(getattr(mod_imported, bp_attr))
+
+    def create_blueprint(self, *args, **kwargs):
+        # Make a flask blueprint having a template folder configured
+        kwargs['template_folder'] = kwargs.get('template_folder', self.template_folder)
+        bp = flask.Blueprint(*args, **kwargs)
+
+        return bp
+
+    def create_named_blueprint(self, *args, **kwargs):
+        # Make a flask blueprint named with the component name, having a template folder configured
+        return self.create_blueprint(self.name, *args, **kwargs)
+
+
+class ModelMixin:
+    db_visit_modules = ('.model.entities', )
+
+
+class ViewMixin:
+    load_blueprints = (('.views', 'component_bp'), )
+
+
+class KegModelComponent(ModelMixin, KegComponent):
+    pass
+
+
+class KegViewComponent(ViewMixin, KegComponent):
+    pass
+
+
+class KegModelViewComponent(ModelMixin, ViewMixin, KegComponent):
+    pass
diff --git a/keg_apps/db/blog/__init__.py b/keg_apps/db/blog/__init__.py
new file mode 100644
index 0000000..495a131
--- /dev/null
+++ b/keg_apps/db/blog/__init__.py
@@ -0,0 +1,4 @@
+from keg import KegModelComponent
+
+
+__component__ = KegModelComponent('blog')
diff --git a/keg_apps/db/blog/model/__init__.py b/keg_apps/db/blog/model/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/keg_apps/db/blog/model/entities.py b/keg_apps/db/blog/model/entities.py
new file mode 100644
index 0000000..357673d
--- /dev/null
+++ b/keg_apps/db/blog/model/entities.py
@@ -0,0 +1,17 @@
+from blazeutils.strings import randchars
+
+from keg.db import db
+
+
+class Blog(db.Model):
+    __tablename__ = 'blogs'
+
+    id = db.Column(db.Integer, primary_key=True)
+    title = db.Column(db.Unicode(50), unique=True, nullable=False)
+
+    @classmethod
+    def testing_create(cls):
+        blog = Blog(title=randchars())
+        db.session.add(blog)
+        db.session.commit()
+        return blog
diff --git a/keg_apps/db/config.py b/keg_apps/db/config.py
index 978b7bf..c1024ed 100644
--- a/keg_apps/db/config.py
+++ b/keg_apps/db/config.py
@@ -2,6 +2,10 @@
 class DefaultProfile(object):
     KEG_KEYRING_ENABLE = False
 
+    KEG_REGISTERED_COMPONENTS = {
+        'keg_apps.db.blog',
+    }
+
 
 class TestProfile(object):
     KEG_DB_DIALECT_OPTIONS = {
diff --git a/keg_apps/db/model/entities.py b/keg_apps/db/model/entities.py
index 4d527f3..b7fa703 100644
--- a/keg_apps/db/model/entities.py
+++ b/keg_apps/db/model/entities.py
@@ -3,20 +3,6 @@ from blazeutils.strings import randchars
 from keg.db import db
 
 
-class Blog(db.Model):
-    __tablename__ = 'blogs'
-
-    id = db.Column(db.Integer, primary_key=True)
-    title = db.Column(db.Unicode(50), unique=True, nullable=False)
-
-    @classmethod
-    def testing_create(cls):
-        blog = Blog(title=randchars())
-        db.session.add(blog)
-        db.session.commit()
-        return blog
-
-
 ####
 # The next several entities exist to facilitate testing of the way Keg handles multiple database
 # binds and dialects.
diff --git a/keg_apps/web/app.py b/keg_apps/web/app.py
index a4ed290..7f05058 100644
--- a/keg_apps/web/app.py
+++ b/keg_apps/web/app.py
@@ -13,7 +13,7 @@ def sayfoo():
 
 
 class WebApp(Keg):
-    import_name = __name__
+    import_name = 'keg_apps.web'
     use_blueprints = all_blueprints
     keyring_enabled = False
 
diff --git a/keg_apps/web/blog/__init__.py b/keg_apps/web/blog/__init__.py
new file mode 100644
index 0000000..3f57427
--- /dev/null
+++ b/keg_apps/web/blog/__init__.py
@@ -0,0 +1,3 @@
+from keg import KegViewComponent
+
+__component__ = KegViewComponent('blog')
diff --git a/keg_apps/web/blog/templates/blog.html b/keg_apps/web/blog/templates/blog.html
new file mode 100644
index 0000000..932f350
--- /dev/null
+++ b/keg_apps/web/blog/templates/blog.html
@@ -0,0 +1,1 @@
+I am a blog
diff --git a/keg_apps/web/blog/views.py b/keg_apps/web/blog/views.py
new file mode 100644
index 0000000..729f435
--- /dev/null
+++ b/keg_apps/web/blog/views.py
@@ -0,0 +1,12 @@
+from keg.web import BaseView
+
+from . import __component__
+
+component_bp = __component__.create_named_blueprint(__name__)
+
+
+class Blog(BaseView):
+    blueprint = component_bp
+
+    def get(self):
+        return self.render()
diff --git a/keg_apps/web/config.py b/keg_apps/web/config.py
new file mode 100644
index 0000000..e514ab3
--- /dev/null
+++ b/keg_apps/web/config.py
@@ -0,0 +1,5 @@
+class DefaultProfile(object):
+
+    KEG_REGISTERED_COMPONENTS = {
+        'keg_apps.web.blog',
+    }
diff --git a/readme.rst b/readme.rst
index dff8760..a5a7ea8 100644
--- a/readme.rst
+++ b/readme.rst
@@ -228,6 +228,54 @@ A view may be given a `url` attribute to override the default:
 See `keg_apps/web/views/routing.py` for other routing possibilities that BaseView supports.
 
 
+Components
+==========
+
+Keg components follow the paradigm of flask extensions, and provide some defaults for the
+purpose of setting up model/view structure. Using components, a project may be broken down into
+logical blocks, each having their own entities, blueprints, templates, tests, etc.
+
+* Components need to be registered in config at `KEG_REGISTERED_COMPONENTS`
+
+  * The path given here should be a full dotted path to the top level of the component
+
+    * e.g. `my_app.components.blog`
+
+  * At the top level of the component, `__component__` must be defined as an instance of KegComponent
+
+    * Depending on the needs of the component, model and view discovery may be driven by the subclasses
+      of KegComponent that have path defaults
+    * Examples:
+
+      * `__component__ = KegModelComponent('blog')`
+      * `__component__ = KegViewComponent('blog')`
+      * `__component__ = KegModelViewComponent('blog')`
+
+* Component discovery
+
+  * A component will attempt to load model and blueprints on app init
+  * The default paths relative to the component may be modified or extended on the component's definition
+  * Default model path in "model" components: `.model.entities`
+
+    * Override via the component's `db_visit_modules` list of relative import paths
+
+  * Default blueprint path for "view" components: `.views.component_bp`
+
+    * Use the `create_named_blueprint` or `create_blueprint` helpers on the component's `__component__`
+      to create blueprints with configured template folders
+    * Override via the component's `load_blueprints` list
+
+      * List elements are a tuple of the relative import path and the name of the blueprint attribute
+
+    * Components have their own template stores, in a `templates` folder
+
+      * Override the component's template path via the `template_folder` attribute
+
+  * Paths may also be supplied to the constructor
+
+    * e.g. `__component__ = KegComponent('blog', db_visit_modules=('.somewhere.else', ))`
+
+
 Keg Development
 ===============
 


Test patch:
diff --git a/keg/tests/test_db.py b/keg/tests/test_db.py
index 38d9080..8682b27 100644
--- a/keg/tests/test_db.py
+++ b/keg/tests/test_db.py
@@ -7,6 +7,9 @@ from keg.db import db
 from keg.signals import db_init_pre, db_init_post, db_clear_post, db_clear_pre
 from keg.testing import invoke_command
 
+# Note: we do not want to import Blog directly here. Doing so will affect metadata collection and
+# prevent the tests below from proving that component initialization is loading the model.
+# from keg_apps.db.blog.model.entities import Blog
 import keg_apps.db.model.entities as ents
 from keg_apps.db.app import DBApp
 from keg_apps.db2 import DB2App
@@ -27,11 +30,14 @@ class TestDB(object):
         DBApp.testing_prep()
 
     def test_primary_db_entity(self):
-        assert ents.Blog.query.count() == 0
-        blog = ents.Blog(title=u'foo')
+        # importing Blog here to ensure we're importing after app init, which will only succeed if
+        # the component has loaded properly
+        from keg_apps.db.blog.model.entities import Blog
+        assert Blog.query.count() == 0
+        blog = Blog(title=u'foo')
         db.session.add(blog)
         db.session.commit()
-        assert ents.Blog.query.count() == 1
+        assert Blog.query.count() == 1
 
     def test_postgres_bind_db_entity(self):
         assert ents.PGDud.query.count() == 0
@@ -63,6 +69,10 @@ class TestDatabaseManager(object):
         DBApp.testing_prep()
 
     def test_db_init_with_clear(self):
+        # importing Blog here to ensure we're importing after app init, which will only succeed if
+        # the component has loaded properly
+        from keg_apps.db.blog.model.entities import Blog
+
         # have to use self here to enable the inner functions to adjust an outer-context variable
         self.init_pre_connected = False
         self.init_post_connected = False
@@ -85,8 +95,8 @@ class TestDatabaseManager(object):
         def catch_clear_post(app):
             self.clear_post_connected = True
 
-        ents.Blog.testing_create()
-        assert ents.Blog.query.count() >= 1
+        Blog.testing_create()
+        assert Blog.query.count() >= 1
         ents.PGDud2.testing_create()
         assert ents.PGDud2.query.count() >= 1
 
@@ -95,9 +105,9 @@ class TestDatabaseManager(object):
         # todo: We could check all the intermediate steps, but this is more a functional test
         # for now.  If the record is gone and we can create an new blog post, then we assume the
         # clear and init went ok.
-        assert ents.Blog.query.count() == 0
-        ents.Blog.testing_create()
-        assert ents.Blog.query.count() == 1
+        assert Blog.query.count() == 0
+        Blog.testing_create()
+        assert Blog.query.count() == 1
 
         assert self.init_pre_connected
         assert self.init_post_connected
diff --git a/keg/tests/test_web.py b/keg/tests/test_web.py
index ecb73f7..6facb37 100644
--- a/keg/tests/test_web.py
+++ b/keg/tests/test_web.py
@@ -30,3 +30,11 @@ class TestBlueprintUsage(WebBase):
     def test_blueprint_template_folder(self):
         resp = self.testapp.get('/tanagra/blueprint-test')
         assert 'blueprint template found ok' in resp
+
+
+class TestComponentBlueprint(WebBase):
+    appcls = WebApp
+
+    def test_component_blueprint_loads(self):
+        resp = self.testapp.get('/blog')
+        assert 'I am a blog' in resp