Repository: apache/airflow

Issue:
Deprecate importing hooks from plugins in Airflow 1.10.13
In Airflow 2.0 we should remove the ability to import hooks via the plugin mechanism.

I think we should deprecate adding Operators and Hooks via the Airflow plugin mechanism.

Similar to #9500, but this time for Hooks. (Separate PR as this time we want to keep the ability to register hooks, but for operators even registering should go.)


```
    from airflow.hooks.my_plugin import MyHook
```

can become

```
    from my_plugin import MyHook
```

with no impact on functionality/user code.

There is still a possible reason to "register" Hooks in a plugin -- namely making the connection types available in the Connections screen. (We don't do that currently, but we should, so keeping the ability to register hooks in a plugin makes sense, even if we deprecate/remove the import mechanism)


Discussed on mailing list here https://lists.apache.org/thread.html/a1453d6a6f113709386b61c68c3f5cd61b258fe78f07811169500fe3%40%3Cdev.airflow.apache.org%3E


Discussion:

Resolving patch:
diff --git a/airflow/hooks/__init__.py b/airflow/hooks/__init__.py
--- a/airflow/hooks/__init__.py
+++ b/airflow/hooks/__init__.py
@@ -22,6 +22,8 @@
 import sys
 
 
+PY37 = sys.version_info >= (3, 7)
+
 # ------------------------------------------------------------------------
 #
 # #TODO #FIXME Airflow 2.0
@@ -77,6 +79,10 @@ def _integrate_plugins():
     from airflow.plugins_manager import hooks_modules
     for hooks_module in hooks_modules:
         sys.modules[hooks_module.__name__] = hooks_module
+
+        if not PY37:
+            from pep562 import Pep562
+            hooks_module = Pep562(hooks_module.__name__)
         globals()[hooks_module._name] = hooks_module
 
         ##########################################################
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -209,10 +209,12 @@ def is_valid_plugin(plugin_obj, existing_plugins):
 )
 
 
-def make_deprecated_module(name, objects, plugin):
-    name = name.lower()
+def make_deprecated_module(kind, plugin, objects=None):
+    name = 'airflow.{}.{}'.format(kind, plugin.name)
     module = imp.new_module(name)
     module._name = name.split('.')[-1]
+    if objects is None:
+        objects = getattr(plugin, kind)
     module._objects = objects
     objects = {o.__name__: o for o in objects}
 
@@ -236,12 +238,12 @@ def __getattr__(attrname):
             correct_import_name = '.'.join((obj.__module__, obj_name))
 
         warnings.warn(
-            "Importing '{}' from under 'airflow.operators.*' has been deprecated and should be directly "
+            "Importing '{}' from under 'airflow.{}.*' has been deprecated and should be directly "
             "imported as '{}' instead.\n"
             "\n"
             "Support for importing from within the airflow namespace for plugins will be dropped entirely "
             "in Airflow 2.0. See <http://airflow.apache.org/docs/stable/howto/custom-operator.html>.".format(
-                attrname, correct_import_name
+                attrname, kind, correct_import_name
             ),
             category=FutureWarning,
             stacklevel=stacklevel
@@ -292,11 +294,13 @@ def make_module(name, objects):
 
 for p in plugins:
     operators_modules.append(
-        make_deprecated_module('airflow.operators.' + p.name, p.operators + p.sensors, p))
+        make_deprecated_module('operators', p, p.operators + p.sensors))
     sensors_modules.append(
-        make_deprecated_module('airflow.sensors.' + p.name, p.sensors, p)
+        make_deprecated_module('sensors', p)
+    )
+    hooks_modules.append(
+        make_deprecated_module('hooks', p)
     )
-    hooks_modules.append(make_module('airflow.hooks.' + p.name, p.hooks))
     executors_modules.append(
         make_module('airflow.executors.' + p.name, p.executors))
     macros_modules.append(make_module('airflow.macros.' + p.name, p.macros))


Test patch:
diff --git a/tests/plugins/test_plugins_manager_www.py b/tests/plugins/test_plugins_manager_www.py
--- a/tests/plugins/test_plugins_manager_www.py
+++ b/tests/plugins/test_plugins_manager_www.py
@@ -22,7 +22,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import unittest
+import six
 from mock import MagicMock, PropertyMock
 
 from flask.blueprints import Blueprint
@@ -37,19 +37,39 @@
 
 from tests.plugins.test_plugin import MockPluginA, MockPluginB, MockPluginC
 
+if six.PY2:
+    # Need `assertWarns` back-ported from unittest2
+    import unittest2 as unittest
+else:
+    import unittest
 
-class PluginsTest(unittest.TestCase):
 
+class PluginsTest(unittest.TestCase):
     def test_operators(self):
-        from airflow.operators.test_plugin import PluginOperator
+        with self.assertWarnsRegex(
+            FutureWarning, r"'PluginOperator' from under 'airflow.operators.*'"
+        ):
+            from airflow.operators.test_plugin import PluginOperator
         self.assertTrue(issubclass(PluginOperator, BaseOperator))
 
+        with self.assertWarnsRegex(
+            FutureWarning, r"'PluginSensorOperator' from under 'airflow.operators.*'"
+        ):
+            from airflow.operators.test_plugin import PluginSensorOperator
+        self.assertTrue(issubclass(PluginSensorOperator, BaseSensorOperator))
+
     def test_sensors(self):
-        from airflow.sensors.test_plugin import PluginSensorOperator
+        with self.assertWarnsRegex(
+            FutureWarning, r"'PluginSensorOperator' from under 'airflow.sensors.*'"
+        ):
+            from airflow.sensors.test_plugin import PluginSensorOperator
         self.assertTrue(issubclass(PluginSensorOperator, BaseSensorOperator))
 
     def test_hooks(self):
-        from airflow.hooks.test_plugin import PluginHook
+        with self.assertWarnsRegex(
+            FutureWarning, r"'PluginHook' from under 'airflow.hooks.*'"
+        ):
+            from airflow.hooks.test_plugin import PluginHook
         self.assertTrue(issubclass(PluginHook, BaseHook))
 
     def test_executors(self):