Repository: apache/airflow

Issue:
Remove adding Operators via plugins in 2.0
In Airflow 2.0 we should remove the ability to add operators and hooks via plugins. 

I think we should deprecate adding Operators and Hooks via the Airflow plugin mechanism.

I think plugins should be reserved for any mechanism that a plain-ol python module import won't work for (which is basically anything that needs to tie deeply in to the Webserver or Scheduler process).

To that end I think we should deprecate adding operators via plugins:

```python
    from airflow.operators.my_plugin import MyOperator
```

can become

```python
    from my_plugin import MyOperator
```

with no impact on functionality.

For this not to be a hard/sudden breaking change we should issue a deprecation warning for this in 1.10.12 - https://github.com/apache/airflow/issues/9500


Discussed here:

https://lists.apache.org/thread.html/a1453d6a6f113709386b61c68c3f5cd61b258fe78f07811169500fe3%40%3Cdev.airflow.apache.org%3E



Discussion:
First of all, we should prepare documentation that describes how loading operators works to increase awareness of the problem. Many people use plugins because they know no other alternative.
https://github.com/apache/airflow/issues/8715
Good call. We can adopt something from https://www.astronomer.io/guides/airflow-importing-custom-hooks-operators/
Docs have been added about Import Mechanism: https://github.com/apache/airflow/pull/10303

https://airflow.readthedocs.io/en/latest/modules_management.html

Resolving patch:
diff --git a/airflow/cli/commands/plugins_command.py b/airflow/cli/commands/plugins_command.py
--- a/airflow/cli/commands/plugins_command.py
+++ b/airflow/cli/commands/plugins_command.py
@@ -25,8 +25,6 @@
 PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP = [
     "plugins",
     "import_errors",
-    "operators_modules",
-    "sensors_modules",
     "hooks_modules",
     "macros_modules",
     "executors_modules",
@@ -39,8 +37,6 @@
 ]
 # list to maintain the order of items.
 PLUGINS_ATTRIBUTES_TO_DUMP = [
-    "operators",
-    "sensors",
     "hooks",
     "executors",
     "macros",
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -38,8 +38,6 @@
 plugins = None  # type: Optional[List[AirflowPlugin]]
 
 # Plugin components to integrate as modules
-operators_modules: Optional[List[Any]] = None
-sensors_modules: Optional[List[Any]] = None
 hooks_modules: Optional[List[Any]] = None
 macros_modules: Optional[List[Any]] = None
 executors_modules: Optional[List[Any]] = None
@@ -106,8 +104,6 @@ class AirflowPlugin:
 
     name: Optional[str] = None
     source: Optional[AirflowPluginSource] = None
-    operators: List[Any] = []
-    sensors: List[Any] = []
     hooks: List[Any] = []
     executors: List[Any] = []
     macros: List[Any] = []
@@ -382,18 +378,11 @@ def integrate_dag_plugins() -> None:
     """Integrates operator, sensor, hook, macro plugins."""
     # pylint: disable=global-statement
     global plugins
-    global operators_modules
-    global sensors_modules
     global hooks_modules
     global macros_modules
     # pylint: enable=global-statement
 
-    if (
-        operators_modules is not None
-        and sensors_modules is not None
-        and hooks_modules is not None
-        and macros_modules is not None
-    ):
+    if hooks_modules is not None and macros_modules is not None:
         return
 
     ensure_plugins_loaded()
@@ -403,8 +392,6 @@ def integrate_dag_plugins() -> None:
 
     log.debug("Integrate DAG plugins")
 
-    operators_modules = []
-    sensors_modules = []
     hooks_modules = []
     macros_modules = []
 
@@ -412,19 +399,9 @@ def integrate_dag_plugins() -> None:
         if plugin.name is None:
             raise AirflowPluginException("Invalid plugin name")
 
-        operators_module = make_module(f'airflow.operators.{plugin.name}', plugin.operators + plugin.sensors)
-        sensors_module = make_module(f'airflow.sensors.{plugin.name}', plugin.sensors)
         hooks_module = make_module(f'airflow.hooks.{plugin.name}', plugin.hooks)
         macros_module = make_module(f'airflow.macros.{plugin.name}', plugin.macros)
 
-        if operators_module:
-            operators_modules.append(operators_module)
-            sys.modules[operators_module.__name__] = operators_module  # pylint: disable=no-member
-
-        if sensors_module:
-            sensors_modules.append(sensors_module)
-            sys.modules[sensors_module.__name__] = sensors_module  # pylint: disable=no-member
-
         if hooks_module:
             hooks_modules.append(hooks_module)
             sys.modules[hooks_module.__name__] = hooks_module  # pylint: disable=no-member
diff --git a/airflow/www/views.py b/airflow/www/views.py
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2851,8 +2851,6 @@ class PluginView(AirflowBaseView):
     ]
 
     plugins_attributes_to_dump = [
-        "operators",
-        "sensors",
         "hooks",
         "executors",
         "macros",


Test patch:
diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py
--- a/tests/cli/commands/test_plugins_command.py
+++ b/tests/cli/commands/test_plugins_command.py
@@ -59,4 +59,3 @@ def test_should_display_one_plugins(self):
             stdout = temp_stdout.getvalue()
         self.assertIn('plugins = [<class ', stdout)
         self.assertIn('test-plugin-cli', stdout)
-        self.assertIn('test_plugins_command.PluginOperator', stdout)
diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -108,18 +108,11 @@ class AirflowTestPropertyPlugin(AirflowPlugin):
             name = "test_property_plugin"
 
             @property
-            def operators(self):
-                from airflow.models.baseoperator import BaseOperator
-
-                class PluginPropertyOperator(BaseOperator):
+            def hooks(self):
+                class TestPropertyHook(BaseHook):
                     pass
 
-                return [PluginPropertyOperator]
-
-            class TestNonPropertyHook(BaseHook):
-                pass
-
-            hooks = [TestNonPropertyHook]
+                return [TestPropertyHook]
 
         with mock_plugin_manager(plugins=[AirflowTestPropertyPlugin()]):
             from airflow import plugins_manager
@@ -127,8 +120,7 @@ class TestNonPropertyHook(BaseHook):
             plugins_manager.integrate_dag_plugins()
 
             self.assertIn('AirflowTestPropertyPlugin', str(plugins_manager.plugins))
-            self.assertIn('PluginPropertyOperator', str(plugins_manager.operators_modules[0].__dict__))
-            self.assertIn("TestNonPropertyHook", str(plugins_manager.hooks_modules[0].__dict__))
+            self.assertIn("TestPropertyHook", str(plugins_manager.hooks_modules[0].__dict__))
 
     def test_should_warning_about_incompatible_plugins(self):
         class AirflowAdminViewsPlugin(AirflowPlugin):
diff --git a/tests/test_utils/mock_plugins.py b/tests/test_utils/mock_plugins.py
--- a/tests/test_utils/mock_plugins.py
+++ b/tests/test_utils/mock_plugins.py
@@ -20,8 +20,6 @@
 
 PLUGINS_MANAGER_NULLABLE_ATTRIBUTES = [
     "plugins",
-    "operators_modules",
-    "sensors_modules",
     "hooks_modules",
     "macros_modules",
     "executors_modules",