Repository: apache/airflow

Issue:
AirflowMacroPluginRemovedRule fails on non-python files
**Apache Airflow version**: 1.10.14


**Kubernetes version (if you are using kubernetes)** (use `kubectl version`):

**Environment**:

- **Cloud provider or hardware configuration**: X
- **OS** (e.g. from /etc/os-release): X
- **Kernel** (e.g. `uname -a`): X
- **Install tools**: X
- **Others**: X

**What happened**:

The `AirflowMacroPluginRemovedRule` seems unable to process non-standard python files (e.g. `.xlsx`) and chokes out with an unhelpful error message.:

```python
========================================================================================================================================================== STATUS ==========================================================================================================================================================

Check for latest versions of apache-airflow and checker...........................................................................................................................................................................................................................................................SUCCESS
Traceback (most recent call last):
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/bin/airflow", line 37, in <module>
    args.func(args)
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/checker.py", line 88, in run
    all_problems = check_upgrade(formatter, rules)
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/checker.py", line 37, in check_upgrade
    rule_status = RuleStatus.from_rule(rule)
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/problem.py", line 44, in from_rule
    result = rule.check()
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/rules/airflow_macro_plugin_removed.py", line 52, in check
    problems.extend(self._check_file(file_path))
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/site-packages/airflow/upgrade/rules/airflow_macro_plugin_removed.py", line 42, in _check_file
    for line_number, line in enumerate(file_pointer, 1):
  File "/Users/madison/programs/anaconda3/envs/memphis-airflow/lib/python3.8/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x82 in position 16: invalid start byte
```

**What you expected to happen**:

I expected the macro to skip over files it could not process/understand

**How to reproduce it**:

Add an `.xlsx` or other binary document to the DAGs folder and run the upgrade check.


**Suggested resolution**:

I think it's fine to fail out on these files (it led us to add certain items to the `.airflowignore` which should have been there anyway) but I had to modify the upgrade rule directly to tell me _which_ files were the problem. A more helpful error message here, and possibly a message prompting users to add said files to their `.airflowignore` would be ideal.



Discussion:
Please feel free to assign this to me too 🙂 
Sure :). Do you want to take care about #13350 too ?
I can give it a shot too! I know less about what the best approach is for that one. Also do you mind adding the `upgrade_check` label to these two?

Resolving patch:
diff --git a/airflow/upgrade/rules/airflow_macro_plugin_removed.py b/airflow/upgrade/rules/airflow_macro_plugin_removed.py
--- a/airflow/upgrade/rules/airflow_macro_plugin_removed.py
+++ b/airflow/upgrade/rules/airflow_macro_plugin_removed.py
@@ -39,9 +39,12 @@ def _check_file(self, file_path):
         problems = []
         class_name_to_check = self.MACRO_PLUGIN_CLASS.split(".")[-1]
         with open(file_path, "r") as file_pointer:
-            for line_number, line in enumerate(file_pointer, 1):
-                if class_name_to_check in line:
-                    problems.append(self._change_info(file_path, line_number))
+            try:
+                for line_number, line in enumerate(file_pointer, 1):
+                    if class_name_to_check in line:
+                        problems.append(self._change_info(file_path, line_number))
+            except UnicodeDecodeError:
+                problems.append("Unable to read python file {}".format(file_path))
         return problems
 
     def check(self):
@@ -49,5 +52,7 @@ def check(self):
         file_paths = list_py_file_paths(directory=dag_folder, include_examples=False)
         problems = []
         for file_path in file_paths:
+            if not file_path.endswith(".py"):
+                continue
             problems.extend(self._check_file(file_path))
         return problems


Test patch:
diff --git a/tests/upgrade/rules/test_airflow_macro_plugin_removed.py b/tests/upgrade/rules/test_airflow_macro_plugin_removed.py
--- a/tests/upgrade/rules/test_airflow_macro_plugin_removed.py
+++ b/tests/upgrade/rules/test_airflow_macro_plugin_removed.py
@@ -20,12 +20,14 @@
 from tempfile import NamedTemporaryFile
 from tests.compat import mock
 
-from airflow.upgrade.rules.airflow_macro_plugin_removed import AirflowMacroPluginRemovedRule
+from airflow.upgrade.rules.airflow_macro_plugin_removed import (
+    AirflowMacroPluginRemovedRule,
+)
 
 
 @contextmanager
-def create_temp_file(mock_list_files, lines):
-    with NamedTemporaryFile("w+") as temp_file:
+def create_temp_file(mock_list_files, lines, extension=".py"):
+    with NamedTemporaryFile("w+", suffix=extension) as temp_file:
         mock_list_files.return_value = [temp_file.name]
         temp_file.writelines("\n".join(lines))
         temp_file.flush()
@@ -34,13 +36,16 @@ def create_temp_file(mock_list_files, lines):
 
 @mock.patch("airflow.upgrade.rules.airflow_macro_plugin_removed.list_py_file_paths")
 class TestAirflowMacroPluginRemovedRule(TestCase):
+
+    def test_rule_info(self, mock_list_files):
+        rule = AirflowMacroPluginRemovedRule()
+        assert isinstance(rule.description, str)
+        assert isinstance(rule.title, str)
+
     def test_valid_check(self, mock_list_files):
         lines = ["import foo.bar.baz"]
         with create_temp_file(mock_list_files, lines):
             rule = AirflowMacroPluginRemovedRule()
-            assert isinstance(rule.description, str)
-            assert isinstance(rule.title, str)
-
             msgs = rule.check()
             assert 0 == len(msgs)
 
@@ -52,10 +57,6 @@ def test_invalid_check(self, mock_list_files):
         with create_temp_file(mock_list_files, lines) as temp_file:
 
             rule = AirflowMacroPluginRemovedRule()
-
-            assert isinstance(rule.description, str)
-            assert isinstance(rule.title, str)
-
             msgs = rule.check()
             assert 2 == len(msgs)
 
@@ -66,3 +67,27 @@ def test_invalid_check(self, mock_list_files):
                 "{} (line {})".format(base_message, line_number) for line_number in [1, 2]
             ]
             assert expected_messages == msgs
+
+    def test_non_python_file_ignored(self, mock_list_files):
+        lines = [
+            "import airflow.AirflowMacroPlugin",
+            "from airflow import AirflowMacroPlugin",
+        ]
+        with create_temp_file(mock_list_files, lines, extension=".other"):
+            # Although this file "matches", it shouldn't be flagged because
+            # only python files are checked for macros anyway
+            rule = AirflowMacroPluginRemovedRule()
+            msgs = rule.check()
+            assert 0 == len(msgs)
+
+    def test_bad_file_failure(self, mock_list_files):
+        # Write a binary file
+        with NamedTemporaryFile("wb+", suffix=".py") as temp_file:
+            mock_list_files.return_value = [temp_file.name]
+            temp_file.write(b"{\x03\xff\x00d")
+            temp_file.flush()
+
+            rule = AirflowMacroPluginRemovedRule()
+            msgs = rule.check()
+            assert 1 == len(msgs)
+            assert msgs == ["Unable to read python file {}".format(temp_file.name)]