Repository: apache/airflow

Issue:
Create NoAdditionalArgsInOperatorsRule to ease upgrade to Airflow 2.0

This issue is part of #8765

## Rule
Create `NoAdditionalArgsInOperatorsRule` which corresponds to
> Additional arguments passed to BaseOperator cause an exception

entry in UPDATING.md. This rule should allow users to check if their current configuration needs any adjusting
before migration to Airflow 2.0.

## How to guide
To implement a new rule, create a class that inherits from `airflow.upgrade.rules.base_rule.BaseRule`.
It will be auto-registered and used by `airflow upgrade-check` command. The custom rule class has to have `title`,
`description` properties and should implement `check` method which returns a list of error messages in case of
incompatibility.

For example:
https://github.com/apache/airflow/blob/ea36166961ca35fc51ddc262ec245590c3e236fb/airflow/upgrade/rules/conn_type_is_not_nullable.py#L25-L42

**Remember** to open the PR against `v1-10-test` branch.



Discussion:
@turbaszek - I would like to give this a shot in the next week or two here.
Hey @coopergillan any update on this one?
I will like to work on this task.

Resolving patch:
diff --git a/airflow/upgrade/rules/no_additional_args_in_operators.py b/airflow/upgrade/rules/no_additional_args_in_operators.py
new file mode 100644
--- /dev/null
+++ b/airflow/upgrade/rules/no_additional_args_in_operators.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+
+import os
+import re
+import logging
+import warnings
+from airflow.utils.dag_processing import correct_maybe_zipped, list_py_file_paths
+from airflow import conf
+from airflow.models.dagbag import DagBag
+from airflow.upgrade.rules.base_rule import BaseRule
+
+
+class NoAdditionalArgsInOperatorsRule(BaseRule):
+    title = "No additional argument allowed in BaseOperator."
+
+    description = """\
+Passing unrecognized arguments to operators is not allowed in Airflow 2.0 anymore,
+and will cause an exception.
+                  """
+
+    def check(self, dags_folder=None):
+        if not dags_folder:
+            dags_folder = conf.get("core", "dags_folder")
+
+        logger = logging.root
+        old_level = logger.level
+        try:
+            logger.setLevel(logging.ERROR)
+            dagbag = DagBag(dag_folder=os.devnull, include_examples=False, store_serialized_dags=False)
+            dags_folder = correct_maybe_zipped(dags_folder)
+
+            # Each file in the DAG folder is parsed individually
+            for filepath in list_py_file_paths(dags_folder,
+                                               safe_mode=conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE'),
+                                               include_examples=False):
+                try:
+                    with warnings.catch_warnings(record=True) as captured_warnings:
+                        _ = dagbag.process_file(
+                            filepath,
+                            safe_mode=conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE'))
+                except Exception:
+                    pass
+
+                for warning in captured_warnings:
+                    if warning.category in (DeprecationWarning, PendingDeprecationWarning) \
+                            and str(warning.message).startswith("Invalid arguments were passed"):
+                        m = re.match(r'''
+                                    .* \(task_id:\ ([^\)]+)\) .* \n
+                                    \*args:\ (.*) \n
+                                     \*\*kwargs:\ (.*)
+                                    ''', str(warning.message), re.VERBOSE)
+
+                        yield "DAG file `{}` with task_id `{}` has unrecognized positional args `{}`" \
+                              "and keyword args " \
+                              "`{}`".format(filepath, m.group(1), m.group(2), m.group(3))
+        finally:
+            logger.setLevel(old_level)


Test patch:
diff --git a/tests/upgrade/rules/test_no_additional_args_operators.py b/tests/upgrade/rules/test_no_additional_args_operators.py
new file mode 100644
--- /dev/null
+++ b/tests/upgrade/rules/test_no_additional_args_operators.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import textwrap
+from tempfile import NamedTemporaryFile
+from unittest import TestCase
+
+import pytest
+
+from airflow.upgrade.rules.no_additional_args_in_operators import NoAdditionalArgsInOperatorsRule
+
+
+class TestNoAdditionalArgsInOperatorsRule(TestCase):
+
+    @pytest.mark.filterwarnings('always')
+    def test_check(self):
+        rule = NoAdditionalArgsInOperatorsRule()
+
+        assert isinstance(rule.title, str)
+        assert isinstance(rule.description, str)
+
+        with NamedTemporaryFile(mode='w', suffix='.py') as dag_file:
+            dag_file.write(
+                textwrap.dedent('''
+            from airflow import DAG
+            from airflow.utils.dates import days_ago
+            from airflow.operators.bash_operator import BashOperator
+
+            with DAG(dag_id="test_no_additional_args_operators", start_date=days_ago(0)):
+                BashOperator(task_id='test', bash_command="true", extra_param=42)
+                '''))
+            dag_file.flush()
+            msgs = list(rule.check(dags_folder=dag_file.name))
+            assert len(msgs) == 1