Repository: apache/airflow

Issue:
Production Docker Image AIRFLOW_INSTALL_VERSION does not overwrite AIRFLOW_VERSION environment variable
**Apache Airflow version**: 1.10.10

**What happened**:
When I look at environment variables within my Airflow containers, I see `AIRFLOW_VERSION="2.0.0.dev0"`

**What you expected to happen**:
Since I built the image with `AIRFLOW_INSTALL_VERSION="==1.10.10" `, I would expect any reference to the Airflow version to reflect 1.10.10 and not 2.0.0.

**How to reproduce it**:
Build an image with a specific AIRFLOW_INSTALL_VERSION and then exec into it and printenv or inspect the container and you will see `AIRFLOW_VERSION="2.0.0.dev0"`. It does not appear like this env var impacts anything and it is hard coded to 2.0.0.dev0 at the moment. Most likely, this variable should be replaced with the AIRFLOW_INSTALL_VERSION instead.

```
docker build . \
--build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.10" 
```

/label area:production-image


Discussion:
Thanks for opening your first issue here! Be sure to follow the issue template!

I finally looked at it - there is not an easy way to do it automatically. I just updated breeze to automatically set, and updated the documentation and examples to clarify that whenever you run AIRFLOW_VERSION, you should also specify AIRFOW_VERSION build arg. PR #12875 

Resolving patch:
diff --git a/.github/workflows/delete_old_artifacts.yml b/airflow/contrib/executors/__init__.py
similarity index 64%
rename from .github/workflows/delete_old_artifacts.yml
rename to airflow/contrib/executors/__init__.py
--- a/.github/workflows/delete_old_artifacts.yml
+++ b/airflow/contrib/executors/__init__.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,18 +16,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
----
-name: 'Delete old artifacts'
-on:  # yamllint disable-line rule:truthy
-  schedule:
-    - cron: '27 */6 * * *'  # run every 6 hours
-
-jobs:
-  delete-artifacts:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: kolpav/purge-artifacts-action@04c636a505f26ebc82f8d070b202fb87ff572b10  # v1.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          expire-in: 7days  # Setting this to 0 will delete all artifacts
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -192,6 +192,9 @@
     '_api/airflow/configuration',
     '_api/airflow/contrib/auth',
     '_api/airflow/contrib/example_dags',
+    '_api/airflow/contrib/executors/index.rst',
+    '_api/airflow/contrib/executors/kubernetes_executor/index.rst',
+    '_api/airflow/contrib/executors/mesos_executor/index.rst',
     '_api/airflow/contrib/index.rst',
     '_api/airflow/contrib/kubernetes',
     '_api/airflow/contrib/task_runner',
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -345,7 +345,6 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])):
     'pyarrow<1.0.0',
     'fsspec<0.8.0;python_version=="3.5"',
     'black==20.8b0;python_version>="3.6"'  # we need to limit black version as we have click < 7
-
 ]
 password = [
     'bcrypt>=2.0.0',
@@ -602,6 +601,7 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])):
     # cattrs >= 1.1.0 dropped support for Python 3.6
     'cattrs>=1.0, <1.1.0;python_version<="3.6"',
     'cattrs>=1.0, <2.0;python_version>"3.6"',
+    'click<8.0.0;python_version<"3.0"',  # click >8 is python 3.6 only but not marked as such yet
     'colorlog==4.0.2',
     'configparser>=3.5.0, <3.6.0',
     'croniter>=0.3.17, <0.4',
@@ -622,7 +622,7 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])):
     'future>=0.16.0, <0.19',
     'graphviz>=0.12',
     'gunicorn>=19.5.0, <21.0',
-    'importlib-metadata~=2.0; python_version<"3.8"',
+    'importlib-metadata~=2.0; python_version<"3.9"',
     'importlib_resources~=1.4',
     'iso8601>=0.1.12',
     'jinja2>=2.10.1, <2.12.0',


Test patch:
diff --git a/chart/tests/helm_template_generator.py b/chart/tests/helm_template_generator.py
--- a/chart/tests/helm_template_generator.py
+++ b/chart/tests/helm_template_generator.py
@@ -61,7 +61,7 @@ def validate_k8s_object(instance):
     validate.validate(instance)
 
 
-def render_chart(name="RELEASE-NAME", values=None, show_only=None):
+def render_chart(name="RELEASE-NAME", values=None, show_only=None, validate_schema=True):
     """
     Function that renders a helm chart into dictionaries. For helm chart testing only
     """
@@ -77,8 +77,9 @@ def render_chart(name="RELEASE-NAME", values=None, show_only=None):
         templates = subprocess.check_output(command)
         k8s_objects = yaml.load_all(templates)
         k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object]  # type: ignore
-        for k8s_object in k8s_objects:
-            validate_k8s_object(k8s_object)
+        if validate_schema:
+            for k8s_object in k8s_objects:
+                validate_k8s_object(k8s_object)
         return k8s_objects
 
 
diff --git a/chart/tests/test_basic_helm_chart.py b/chart/tests/test_basic_helm_chart.py
--- a/chart/tests/test_basic_helm_chart.py
+++ b/chart/tests/test_basic_helm_chart.py
@@ -22,7 +22,7 @@
 
 from tests.helm_template_generator import render_chart
 
-OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 22
+OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 24
 
 
 class TestBaseChartTest(unittest.TestCase):
@@ -50,7 +50,9 @@ def test_basic_deployments(self):
                 ('Secret', 'TEST-BASIC-airflow-result-backend'),
                 ('ConfigMap', 'TEST-BASIC-airflow-config'),
                 ('Role', 'TEST-BASIC-pod-launcher-role'),
+                ('Role', 'TEST-BASIC-pod-log-reader-role'),
                 ('RoleBinding', 'TEST-BASIC-pod-launcher-rolebinding'),
+                ('RoleBinding', 'TEST-BASIC-pod-log-reader-rolebinding'),
                 ('Service', 'TEST-BASIC-postgresql-headless'),
                 ('Service', 'TEST-BASIC-postgresql'),
                 ('Service', 'TEST-BASIC-statsd'),
diff --git a/chart/tests/test_keda.py b/chart/tests/test_keda.py
new file mode 100644
--- /dev/null
+++ b/chart/tests/test_keda.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+import jmespath
+from parameterized import parameterized
+
+from tests.helm_template_generator import render_chart
+
+
+class KedaTest(unittest.TestCase):
+    def test_keda_disabled_by_default(self):
+        """disabled by default"""
+        docs = render_chart(
+            values={},
+            show_only=["templates/workers/worker-kedaautoscaler.yaml"],
+            validate_schema=False,
+        )
+        self.assertListEqual(docs, [])
+
+    @parameterized.expand(
+        [
+            ('SequentialExecutor', False),
+            ('CeleryExecutor', True),
+            ('CeleryKubernetesExecutor', True),
+        ]
+    )
+    def test_keda_enabled(self, executor, is_created):
+        """
+        ScaledObject should only be created when set to enabled and executor is Celery or CeleryKubernetes
+        """
+        docs = render_chart(
+            values={
+                "workers": {"keda": {"enabled": True}, "persistence": {"enabled": False}},
+                'executor': executor,
+            },
+            show_only=["templates/workers/worker-kedaautoscaler.yaml"],
+            validate_schema=False,
+        )
+        if is_created:
+            self.assertEqual("RELEASE-NAME-worker", jmespath.search("metadata.name", docs[0]))
+        else:
+            self.assertListEqual(docs, [])
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -103,15 +103,12 @@ build_images::rebuild_ci_image_if_needed
 
 initialization::set_mysql_encoding
 
-DOCKER_COMPOSE_LOCAL=()
+DOCKER_COMPOSE_LOCAL=("-f" "${SCRIPTS_CI_DIR}/docker-compose/files.yml")
 
 if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then
     DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml")
 fi
 
-if [[ ${MOUNT_FILES} == "true" ]]; then
-    DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/files.yml")
-fi
 
 if [[ ${GITHUB_ACTIONS} == "true" ]]; then
     DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/ga.yml")
diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh
--- a/scripts/in_container/run_ci_tests.sh
+++ b/scripts/in_container/run_ci_tests.sh
@@ -38,46 +38,53 @@ elif [[ "${RES}" != "0" ]]; then
     elif [[ ${BACKEND} == "mysql" ]]; then
         EXTRA_ARGS="--mysql-version ${MYSQL_VERSION} "
     fi
-
-    >&2 echo "***********************************************************************************************"
-    >&2 echo "*"
-    >&2 echo "* ERROR! Some tests failed, unfortunately. Those might be transient errors,"
-    >&2 echo "*        but usually you have to fix something."
-    >&2 echo "*        See the above log for details."
-    >&2 echo "*"
-    >&2 echo "***********************************************************************************************"
-    >&2 echo "*  You can easily reproduce the failed tests on your dev machine/"
-    >&2 echo "*"
-    >&2 echo "*   When you have the source branch checked out locally:"
-    >&2 echo "*"
-    >&2 echo "*     Run all tests:"
-    >&2 echo "*"
-    >&2 echo "*       ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE}  tests"
-    >&2 echo "*"
-    >&2 echo "*     Enter docker shell:"
-    >&2 echo "*"
-    >&2 echo "*       ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE}  shell"
-    >&2 echo "*"
+    echo """
+${COLOR_RED_ERROR}
+***********************************************************************************************
+*
+* ERROR! Some tests failed, unfortunately. Those might be transient errors,
+*        but usually you have to fix something.
+*        See the above log for details.
+*
+***********************************************************************************************
+*  You can easily reproduce the failed tests on your dev machine/
+*
+*   When you have the source branch checked out locally:
+*
+*     Run all tests:
+*
+*       ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE}  tests
+*
+*     Enter docker shell:
+*
+*       ./breeze --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE}  shell
+*
+"""
     if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG=} != "" ]]; then
-        >&2 echo "*   When you do not have sources:"
-        >&2 echo "*"
-        >&2 echo "*     Run all tests:"
-        >&2 echo "*"
-        >&2 echo "*      ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} tests"
-        >&2 echo "*"
-        >&2 echo "*     Enter docker shell:"
-        >&2 echo "*"
-        >&2 echo "*      ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} shell"
-        >&2 echo "*"
+        echo """
+*   When you do not have sources:
+*
+*     Run all tests:
+*
+*      ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} tests
+*
+*     Enter docker shell:
+*
+*      ./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} shell
+*
+"""
     fi
-    >&2 echo "*"
-    >&2 echo "*   NOTE! Once you are in the docker shell, you can run failed test with:"
-    >&2 echo "*"
-    >&2 echo "*            pytest [TEST_NAME]"
-    >&2 echo "*"
-    >&2 echo "*   You can copy the test name from the output above"
-    >&2 echo "*"
-    >&2 echo "***********************************************************************************************"
+    echo """
+*
+*   NOTE! Once you are in the docker shell, you can run failed test with:
+*
+*            pytest [TEST_NAME]
+*
+*   You can copy the test name from the output above
+*
+***********************************************************************************************
+${COLOR_RESET}
+"""
 fi
 
 MAIN_GITHUB_REPOSITORY="apache/airflow"
diff --git a/scripts/in_container/run_system_tests.sh b/scripts/in_container/run_system_tests.sh
--- a/scripts/in_container/run_system_tests.sh
+++ b/scripts/in_container/run_system_tests.sh
@@ -25,6 +25,8 @@ IN_CONTAINER_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
 # shellcheck source=scripts/in_container/_in_container_utils.sh
 . "${IN_CONTAINER_DIR}/_in_container_utils.sh"
 
+in_container_set_colors
+
 in_container_basic_sanity_check
 
 in_container_script_start
diff --git a/tests/bats/breeze/test_breeze_complete.bats b/tests/bats/breeze/test_breeze_complete.bats
--- a/tests/bats/breeze/test_breeze_complete.bats
+++ b/tests/bats/breeze/test_breeze_complete.bats
@@ -272,3 +272,11 @@
 
   assert_equal "" "${TEST_TYPE}"
 }
+
+@test "Test default package format is wheel" {
+  load ../bats_utils
+  #shellcheck source=breeze-complete
+  source "${AIRFLOW_SOURCES}/breeze-complete"
+
+  assert_equal "wheel" "${PACKAGE_FORMAT}"
+}
diff --git a/tests/bats/breeze/test_breeze_params.bats b/tests/bats/breeze/test_breeze_params.bats
--- a/tests/bats/breeze/test_breeze_params.bats
+++ b/tests/bats/breeze/test_breeze_params.bats
@@ -31,10 +31,7 @@ teardown() {
 @test "Test missing value for a parameter" {
   export _breeze_allowed_test_params="a b c"
   run parameters::check_and_save_allowed_param "TEST_PARAM"  "Test Param" "--message"
-  assert_output "
-ERROR:  Allowed Test Param: [ a b c ]. Passed: ''.
-
-Switch to supported value with --message flag."
+  assert_output --regexp "Allowed Test Param: \[ a b c \]\. Passed: ''"
   assert_failure
 }
 
@@ -43,10 +40,7 @@ Switch to supported value with --message flag."
   export TEST_PARAM=x
   echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
   run parameters::check_and_save_allowed_param "TEST_PARAM"  "Test Param" "--message"
-  assert_output "
-ERROR:  Allowed Test Param: [ a b c ]. Passed: 'x'.
-
-Switch to supported value with --message flag."
+  assert_output --regexp "Allowed Test Param: \[ a b c \]\. Passed: 'x"
   assert_exist "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
   assert_file_contains "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" "^a$"
   assert_failure 1
@@ -57,12 +51,7 @@ Switch to supported value with --message flag."
   export TEST_PARAM=x
   echo "x" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
   run parameters::check_and_save_allowed_param "TEST_PARAM"  "Test Param" "--message"
-  assert_output "
-ERROR:  Allowed Test Param: [ a b c ]. Passed: 'x'.
-
-Switch to supported value with --message flag.
-
-Removing ${AIRFLOW_SOURCES}/.build/.TEST_PARAM. Next time you run it, it should be OK."
+  assert_output --regexp "Allowed Test Param: \[ a b c \]\. Passed: 'x'"
   assert_not_exist "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
   assert_failure 1
 }
diff --git a/tests/bats/in_container/test_in_container.bats b/tests/bats/in_container/test_in_container.bats
--- a/tests/bats/in_container/test_in_container.bats
+++ b/tests/bats/in_container/test_in_container.bats
@@ -23,6 +23,8 @@ source "/opt/bats/lib/load.bash"
 setup() {
 # shellcheck source=scripts/in_container/_in_container_utils.sh
     source "${AIRFLOW_SOURCES}/scripts/in_container/_in_container_utils.sh"
+
+    in_container_set_colors
 }
 
 @test "test in_container" {
diff --git a/tests/upgrade/test_formattes.py b/tests/upgrade/test_formattes.py
--- a/tests/upgrade/test_formattes.py
+++ b/tests/upgrade/test_formattes.py
@@ -16,12 +16,13 @@
 # under the License.
 import json
 from tempfile import NamedTemporaryFile
-from tests.compat import mock
 
 import pytest
 
 from airflow.bin import cli
+from airflow.upgrade.checker import run
 from airflow.upgrade.rules.base_rule import BaseRule
+from tests.compat import mock
 
 MESSAGES = ["msg1", "msg2"]
 
@@ -62,7 +63,7 @@ def test_output(self, _):
         with NamedTemporaryFile("w+", suffix=".json") as temp:
             with pytest.raises(SystemExit):
                 args = parser.parse_args(['upgrade_check', '-s', temp.name])
-                args.func(args)
+                run(args)
             content = temp.read()
 
         assert json.loads(content) == expected
diff --git a/tests/www_rbac/test_views.py b/tests/www_rbac/test_views.py
--- a/tests/www_rbac/test_views.py
+++ b/tests/www_rbac/test_views.py
@@ -417,9 +417,9 @@ def test_index(self):
     def test_doc_site_url(self):
         resp = self.client.get('/', follow_redirects=True)
         if "dev" in version.version:
-            airflow_doc_site = "https://airflow.readthedocs.io/en/latest"
+            airflow_doc_site = "https://s.apache.org/airflow-docs"
         else:
-            airflow_doc_site = 'https://airflow.apache.org/docs/{}'.format(version.version)
+            airflow_doc_site = 'https://airflow.apache.org/docs/apache-airflow/{}'.format(version.version)
 
         self.check_content_in_response(airflow_doc_site, resp)