Repository: apache/airflow

Issue:
db+ string in result backend but not metadata secret
### Official Helm Chart version

1.1.0 (latest released)

### Apache Airflow version

2.1.3 (latest released)

### Kubernetes Version

1.21

### Helm Chart configuration

  data:
    metadataSecretName: "airflow-metadata"
    resultBackendSecretName: "airflow-result-backend"

### Docker Image customisations

_No response_

### What happened

If we only supply 1 secret with 
```
connection: postgresql://airflow:password@postgres.rds:5432/airflow?sslmode=disable
```
To use for both metadata and resultBackendConnection then we end up with a connection error because
resultBackendConnection expects the string to be formatted like 
```
connection: db+postgresql://airflow:password@postgres.rds:5432/airflow?sslmode=disable
```
from what i can tell

### What you expected to happen

I'd expect to be able to use the same secret for both using the same format if they are using the same connection. 

### How to reproduce

Make a secret structured like above to look like the metadataConnection auto-generated secret.
use that same secret for the result backend.
deploy.


### Anything else

Occurs always. 
To get around currently we make 2 secrets one with just the db+ prepended. 

### Are you willing to submit PR?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)



Discussion:
I am not sure this is a helm-chart issue. It could probably be handled by the airflow itself - there are just few prefixes of the URL that could be shared with Airflow Metadata DB (postgres/sqlite/mysql variants) and it could be as easy of adding `db+` in front.

@EliMor  - maybe you would like to make a PR in Airflow to implement it? Sounds like an easy thing for a small, nice contribution.
@potiuk  Can I take this? I will submit PR.
I'd be happy to try to take this, but if @bigyan-kc  would like to I don't want to stand in the way. 

Thank you.
Feel free @bigyan-kc. @EliMor  - you can review the change :)
Thank you @potiuk 
I was wondering if we could also add a small comment to the values file to hint what the secret created should look like.
Somewhere around here; https://github.com/apache/airflow/blob/main/chart/values.yaml#L242
Could you add that as well @bigyan-kc  please if @potiuk  thinks its worth it? 

Thank you again.
@EliMor sure I will also add comment to the values file.
Hello,
Is this still available?
Hello, I could use a fix for this issue, but the issue hasn't been updated in a while 😟 If no one minds, can I try submitting a PR for it?
Feel free

Resolving patch:
diff --git a/airflow/config_templates/default_celery.py b/airflow/config_templates/default_celery.py
--- a/airflow/config_templates/default_celery.py
+++ b/airflow/config_templates/default_celery.py
@@ -36,6 +36,12 @@ def _broker_supports_visibility_timeout(url):
     if _broker_supports_visibility_timeout(broker_url):
         broker_transport_options['visibility_timeout'] = 21600
 
+if conf.has_option("celery", 'RESULT_BACKEND'):
+    result_backend = conf.get_mandatory_value('celery', 'RESULT_BACKEND')
+else:
+    log.debug("Value for celery result_backend not found. Using sql_alchemy_conn with db+ prefix.")
+    result_backend = f'db+{conf.get("database", "SQL_ALCHEMY_CONN")}'
+
 DEFAULT_CELERY_CONFIG = {
     'accept_content': ['json'],
     'event_serializer': 'json',
@@ -46,7 +52,7 @@ def _broker_supports_visibility_timeout(url):
     'task_track_started': conf.getboolean('celery', 'task_track_started'),
     'broker_url': broker_url,
     'broker_transport_options': broker_transport_options,
-    'result_backend': conf.get('celery', 'RESULT_BACKEND'),
+    'result_backend': result_backend,
     'worker_concurrency': conf.getint('celery', 'WORKER_CONCURRENCY'),
     'worker_enable_remote_control': conf.getboolean('celery', 'worker_enable_remote_control'),
 }
@@ -92,7 +98,6 @@ def _broker_supports_visibility_timeout(url):
         f'all necessary certs and key ({e}).'
     )
 
-result_backend = str(DEFAULT_CELERY_CONFIG['result_backend'])
 if 'amqp://' in result_backend or 'redis://' in result_backend or 'rpc://' in result_backend:
     log.warning(
         "You have configured a result_backend of %s, it is highly recommended "


Test patch:
diff --git a/tests/charts/test_basic_helm_chart.py b/tests/charts/test_basic_helm_chart.py
--- a/tests/charts/test_basic_helm_chart.py
+++ b/tests/charts/test_basic_helm_chart.py
@@ -25,24 +25,44 @@
 
 from tests.charts.helm_template_generator import render_chart
 
-OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 35
+OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 34
 
 
 class TestBaseChartTest(unittest.TestCase):
-    def test_basic_deployments(self):
+    def _get_values_with_version(self, values, version):
+        if version != "default":
+            values["airflowVersion"] = version
+        return values
+
+    def _get_object_count(self, version):
+        # TODO remove default from condition after airflow update
+        if version == "2.3.2" or version == "default":
+            return OBJECT_COUNT_IN_BASIC_DEPLOYMENT + 1
+        return OBJECT_COUNT_IN_BASIC_DEPLOYMENT
+
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_basic_deployments(self, version):
+        expected_object_count_in_basic_deployment = self._get_object_count(version)
         k8s_objects = render_chart(
             "TEST-BASIC",
-            values={
-                "chart": {
-                    'metadata': 'AA',
+            self._get_values_with_version(
+                values={
+                    "chart": {
+                        'metadata': 'AA',
+                    },
+                    'labels': {"TEST-LABEL": "TEST-VALUE"},
+                    "fullnameOverride": "TEST-BASIC",
                 },
-                'labels': {"TEST-LABEL": "TEST-VALUE"},
-                "fullnameOverride": "TEST-BASIC",
-            },
+                version=version,
+            ),
         )
         list_of_kind_names_tuples = {
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
         }
+        # TODO remove default from condition after airflow update
+        if version == "2.3.2" or version == "default":
+            assert ('Secret', 'TEST-BASIC-airflow-result-backend') in list_of_kind_names_tuples
+            list_of_kind_names_tuples.remove(('Secret', 'TEST-BASIC-airflow-result-backend'))
         assert list_of_kind_names_tuples == {
             ('ServiceAccount', 'TEST-BASIC-create-user-job'),
             ('ServiceAccount', 'TEST-BASIC-migrate-database-job'),
@@ -53,7 +73,6 @@ def test_basic_deployments(self):
             ('ServiceAccount', 'TEST-BASIC-webserver'),
             ('ServiceAccount', 'TEST-BASIC-worker'),
             ('Secret', 'TEST-BASIC-airflow-metadata'),
-            ('Secret', 'TEST-BASIC-airflow-result-backend'),
             ('Secret', 'TEST-BASIC-broker-url'),
             ('Secret', 'TEST-BASIC-fernet-key'),
             ('Secret', 'TEST-BASIC-webserver-secret-key'),
@@ -80,7 +99,7 @@ def test_basic_deployments(self):
             ('Job', 'TEST-BASIC-create-user'),
             ('Job', 'TEST-BASIC-run-airflow-migrations'),
         }
-        assert OBJECT_COUNT_IN_BASIC_DEPLOYMENT == len(k8s_objects)
+        assert expected_object_count_in_basic_deployment == len(k8s_objects)
         for k8s_object in k8s_objects:
             labels = jmespath.search('metadata.labels', k8s_object) or {}
             if 'helm.sh/chart' in labels:
@@ -94,16 +113,20 @@ def test_basic_deployments(self):
                 "TEST-LABEL"
             ), f"Missing label TEST-LABEL on {k8s_name}. Current labels: {labels}"
 
-    def test_basic_deployment_without_default_users(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_basic_deployment_without_default_users(self, version):
+        expected_object_count_in_basic_deployment = self._get_object_count(version)
         k8s_objects = render_chart(
             "TEST-BASIC",
-            values={"webserver": {"defaultUser": {'enabled': False}}},
+            values=self._get_values_with_version(
+                values={"webserver": {"defaultUser": {'enabled': False}}}, version=version
+            ),
         )
         list_of_kind_names_tuples = [
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
         ]
         assert ('Job', 'TEST-BASIC-create-user') not in list_of_kind_names_tuples
-        assert OBJECT_COUNT_IN_BASIC_DEPLOYMENT - 2 == len(k8s_objects)
+        assert expected_object_count_in_basic_deployment - 2 == len(k8s_objects)
 
     def test_network_policies_are_valid(self):
         k8s_objects = render_chart(
@@ -139,6 +162,17 @@ def test_labels_are_valid(self):
             values={
                 "labels": {"label1": "value1", "label2": "value2"},
                 "executor": "CeleryExecutor",
+                "data": {
+                    "resultBackendConnection": {
+                        "user": "someuser",
+                        "pass": "somepass",
+                        "host": "somehost",
+                        "protocol": "postgresql",
+                        "port": 7777,
+                        "db": "somedb",
+                        "sslmode": "allow",
+                    }
+                },
                 "pgbouncer": {"enabled": True},
                 "redis": {"enabled": True},
                 "ingress": {"enabled": True},
diff --git a/tests/charts/test_rbac.py b/tests/charts/test_rbac.py
--- a/tests/charts/test_rbac.py
+++ b/tests/charts/test_rbac.py
@@ -18,13 +18,13 @@
 import unittest
 
 import jmespath
+from parameterized import parameterized
 
 from tests.charts.helm_template_generator import render_chart
 
 DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES = [
     ('Secret', 'TEST-RBAC-postgresql'),
     ('Secret', 'TEST-RBAC-airflow-metadata'),
-    ('Secret', 'TEST-RBAC-airflow-result-backend'),
     ('Secret', 'TEST-RBAC-pgbouncer-config'),
     ('Secret', 'TEST-RBAC-pgbouncer-stats'),
     ('ConfigMap', 'TEST-RBAC-airflow-config'),
@@ -105,34 +105,51 @@
 
 
 class RBACTest(unittest.TestCase):
-    def test_deployments_no_rbac_no_sa(self):
+    def _get_values_with_version(self, values, version):
+        if version != "default":
+            values["airflowVersion"] = version
+        return values
+
+    def _get_object_count(self, version):
+        # TODO remove default from condition after airflow update
+        if version == "2.3.2" or version == "default":
+            return [
+                ('Secret', 'TEST-RBAC-airflow-result-backend')
+            ] + DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES
+        return DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES
+
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_deployments_no_rbac_no_sa(self, version):
         k8s_objects = render_chart(
             "TEST-RBAC",
-            values={
-                "fullnameOverride": "TEST-RBAC",
-                "rbac": {"create": False},
-                "cleanup": {
-                    "enabled": True,
-                    "serviceAccount": {
-                        "create": False,
+            values=self._get_values_with_version(
+                values={
+                    "fullnameOverride": "TEST-RBAC",
+                    "rbac": {"create": False},
+                    "cleanup": {
+                        "enabled": True,
+                        "serviceAccount": {
+                            "create": False,
+                        },
                     },
-                },
-                "pgbouncer": {
-                    "enabled": True,
-                    "serviceAccount": {
-                        "create": False,
+                    "pgbouncer": {
+                        "enabled": True,
+                        "serviceAccount": {
+                            "create": False,
+                        },
                     },
+                    "redis": {"serviceAccount": {"create": False}},
+                    "scheduler": {"serviceAccount": {"create": False}},
+                    "webserver": {"serviceAccount": {"create": False}},
+                    "workers": {"serviceAccount": {"create": False}},
+                    "triggerer": {"serviceAccount": {"create": False}},
+                    "statsd": {"serviceAccount": {"create": False}},
+                    "createUserJob": {"serviceAccount": {"create": False}},
+                    "migrateDatabaseJob": {"serviceAccount": {"create": False}},
+                    "flower": {"enabled": True, "serviceAccount": {"create": False}},
                 },
-                "redis": {"serviceAccount": {"create": False}},
-                "scheduler": {"serviceAccount": {"create": False}},
-                "webserver": {"serviceAccount": {"create": False}},
-                "workers": {"serviceAccount": {"create": False}},
-                "triggerer": {"serviceAccount": {"create": False}},
-                "statsd": {"serviceAccount": {"create": False}},
-                "createUserJob": {"serviceAccount": {"create": False}},
-                "migrateDatabaseJob": {"serviceAccount": {"create": False}},
-                "flower": {"enabled": True, "serviceAccount": {"create": False}},
-            },
+                version=version,
+            ),
         )
         list_of_kind_names_tuples = [
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
@@ -140,83 +157,93 @@ def test_deployments_no_rbac_no_sa(self):
 
         self.assertCountEqual(
             list_of_kind_names_tuples,
-            DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES,
+            self._get_object_count(version),
         )
 
-    def test_deployments_no_rbac_with_sa(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_deployments_no_rbac_with_sa(self, version):
         k8s_objects = render_chart(
             "TEST-RBAC",
-            values={
-                "fullnameOverride": "TEST-RBAC",
-                "rbac": {"create": False},
-                "cleanup": {"enabled": True},
-                "flower": {"enabled": True},
-                "pgbouncer": {"enabled": True},
-            },
+            values=self._get_values_with_version(
+                values={
+                    "fullnameOverride": "TEST-RBAC",
+                    "rbac": {"create": False},
+                    "cleanup": {"enabled": True},
+                    "flower": {"enabled": True},
+                    "pgbouncer": {"enabled": True},
+                },
+                version=version,
+            ),
         )
         list_of_kind_names_tuples = [
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
         ]
-        real_list_of_kind_names = DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES + SERVICE_ACCOUNT_NAME_TUPLES
+        real_list_of_kind_names = self._get_object_count(version) + SERVICE_ACCOUNT_NAME_TUPLES
         self.assertCountEqual(
             list_of_kind_names_tuples,
             real_list_of_kind_names,
         )
 
-    def test_deployments_with_rbac_no_sa(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_deployments_with_rbac_no_sa(self, version):
         k8s_objects = render_chart(
             "TEST-RBAC",
-            values={
-                "fullnameOverride": "TEST-RBAC",
-                "cleanup": {
-                    "enabled": True,
-                    "serviceAccount": {
-                        "create": False,
+            values=self._get_values_with_version(
+                values={
+                    "fullnameOverride": "TEST-RBAC",
+                    "cleanup": {
+                        "enabled": True,
+                        "serviceAccount": {
+                            "create": False,
+                        },
                     },
-                },
-                "scheduler": {"serviceAccount": {"create": False}},
-                "webserver": {"serviceAccount": {"create": False}},
-                "workers": {"serviceAccount": {"create": False}},
-                "triggerer": {"serviceAccount": {"create": False}},
-                "flower": {"enabled": True, "serviceAccount": {"create": False}},
-                "statsd": {"serviceAccount": {"create": False}},
-                "redis": {"serviceAccount": {"create": False}},
-                "pgbouncer": {
-                    "enabled": True,
-                    "serviceAccount": {
-                        "create": False,
+                    "scheduler": {"serviceAccount": {"create": False}},
+                    "webserver": {"serviceAccount": {"create": False}},
+                    "workers": {"serviceAccount": {"create": False}},
+                    "triggerer": {"serviceAccount": {"create": False}},
+                    "flower": {"enabled": True, "serviceAccount": {"create": False}},
+                    "statsd": {"serviceAccount": {"create": False}},
+                    "redis": {"serviceAccount": {"create": False}},
+                    "pgbouncer": {
+                        "enabled": True,
+                        "serviceAccount": {
+                            "create": False,
+                        },
                     },
+                    "createUserJob": {"serviceAccount": {"create": False}},
+                    "migrateDatabaseJob": {"serviceAccount": {"create": False}},
                 },
-                "createUserJob": {"serviceAccount": {"create": False}},
-                "migrateDatabaseJob": {"serviceAccount": {"create": False}},
-            },
+                version=version,
+            ),
         )
         list_of_kind_names_tuples = [
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
         ]
-        real_list_of_kind_names = DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES + RBAC_ENABLED_KIND_NAME_TUPLES
+        real_list_of_kind_names = self._get_object_count(version) + RBAC_ENABLED_KIND_NAME_TUPLES
         self.assertCountEqual(
             list_of_kind_names_tuples,
             real_list_of_kind_names,
         )
 
-    def test_deployments_with_rbac_with_sa(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_deployments_with_rbac_with_sa(self, version):
         k8s_objects = render_chart(
             "TEST-RBAC",
-            values={
-                "fullnameOverride": "TEST-RBAC",
-                "cleanup": {"enabled": True},
-                "flower": {"enabled": True},
-                "pgbouncer": {"enabled": True},
-            },
+            values=self._get_values_with_version(
+                values={
+                    "fullnameOverride": "TEST-RBAC",
+                    "cleanup": {"enabled": True},
+                    "flower": {"enabled": True},
+                    "pgbouncer": {"enabled": True},
+                },
+                version=version,
+            ),
         )
         list_of_kind_names_tuples = [
             (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
         ]
         real_list_of_kind_names = (
-            DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES
-            + SERVICE_ACCOUNT_NAME_TUPLES
-            + RBAC_ENABLED_KIND_NAME_TUPLES
+            self._get_object_count(version) + SERVICE_ACCOUNT_NAME_TUPLES + RBAC_ENABLED_KIND_NAME_TUPLES
         )
         self.assertCountEqual(
             list_of_kind_names_tuples,
diff --git a/tests/charts/test_result_backend_connection_secret.py b/tests/charts/test_result_backend_connection_secret.py
--- a/tests/charts/test_result_backend_connection_secret.py
+++ b/tests/charts/test_result_backend_connection_secret.py
@@ -17,6 +17,7 @@
 
 import base64
 import unittest
+from typing import Union
 
 import jmespath
 from parameterized import parameterized
@@ -25,6 +26,17 @@
 
 
 class ResultBackendConnectionSecretTest(unittest.TestCase):
+    def _get_values_with_version(self, values, version):
+        if version != "default":
+            values["airflowVersion"] = version
+        return values
+
+    def _assert_for_old_version(self, version, value, expected_value):
+        # TODO remove default from condition after airflow update
+        if version == "2.3.2" or version == "default":
+            assert value == expected_value
+        else:
+            assert value is None
 
     non_chart_database_values = {
         "user": "someuser",
@@ -53,49 +65,69 @@ def test_should_not_generate_a_document_if_using_existing_secret(self):
     )
     def test_should_a_document_be_generated_for_executor(self, executor, expected_doc_count):
         docs = render_chart(
-            values={"executor": executor},
+            values={
+                "executor": executor,
+                "data": {
+                    "metadataConnection": {**self.non_chart_database_values},
+                    "resultBackendConnection": {
+                        **self.non_chart_database_values,
+                        "user": "anotheruser",
+                        "pass": "anotherpass",
+                    },
+                },
+            },
             show_only=["templates/secrets/result-backend-connection-secret.yaml"],
         )
 
         assert expected_doc_count == len(docs)
 
-    def _get_connection(self, values: dict) -> str:
+    def _get_connection(self, values: dict) -> Union[str, None]:
         docs = render_chart(
             values=values,
             show_only=["templates/secrets/result-backend-connection-secret.yaml"],
         )
+        if len(docs) == 0:
+            return None
         encoded_connection = jmespath.search("data.connection", docs[0])
         return base64.b64decode(encoded_connection).decode()
 
-    def test_default_connection(self):
-        connection = self._get_connection({})
-
-        assert (
-            "db+postgresql://postgres:postgres@RELEASE-NAME-postgresql:5432/postgres?sslmode=disable"
-            == connection
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_default_connection_old_version(self, version):
+        connection = self._get_connection(self._get_values_with_version(version=version, values={}))
+        self._assert_for_old_version(
+            version,
+            value=connection,
+            expected_value="db+postgresql://postgres:postgres@RELEASE-NAME"
+            "-postgresql:5432/postgres?sslmode=disable",
         )
 
-    def test_should_default_to_custom_metadata_db_connection_with_pgbouncer_overrides(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_should_default_to_custom_metadata_db_connection_with_pgbouncer_overrides(self, version):
         values = {
             "pgbouncer": {"enabled": True},
             "data": {"metadataConnection": {**self.non_chart_database_values}},
         }
-        connection = self._get_connection(values)
+        connection = self._get_connection(self._get_values_with_version(values=values, version=version))
 
         # host, port, dbname still get overridden
-        assert (
-            "db+postgresql://someuser:somepass@RELEASE-NAME-pgbouncer:6543"
-            "/RELEASE-NAME-result-backend?sslmode=allow" == connection
+        self._assert_for_old_version(
+            version,
+            value=connection,
+            expected_value="db+postgresql://someuser:somepass@RELEASE-NAME-pgbouncer"
+            ":6543/RELEASE-NAME-result-backend?sslmode=allow",
         )
 
-    def test_should_set_pgbouncer_overrides_when_enabled(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_should_set_pgbouncer_overrides_when_enabled(self, version):
         values = {"pgbouncer": {"enabled": True}}
-        connection = self._get_connection(values)
+        connection = self._get_connection(self._get_values_with_version(values=values, version=version))
 
         # host, port, dbname get overridden
-        assert (
-            "db+postgresql://postgres:postgres@RELEASE-NAME-pgbouncer:6543"
-            "/RELEASE-NAME-result-backend?sslmode=disable" == connection
+        self._assert_for_old_version(
+            version,
+            value=connection,
+            expected_value="db+postgresql://postgres:postgres@RELEASE-NAME-pgbouncer"
+            ":6543/RELEASE-NAME-result-backend?sslmode=disable",
         )
 
     def test_should_set_pgbouncer_overrides_with_non_chart_database_when_enabled(self):
@@ -111,13 +143,17 @@ def test_should_set_pgbouncer_overrides_with_non_chart_database_when_enabled(sel
             "/RELEASE-NAME-result-backend?sslmode=allow" == connection
         )
 
-    def test_should_default_to_custom_metadata_db_connection(self):
+    @parameterized.expand(["2.3.2", "2.4.0", "default"])
+    def test_should_default_to_custom_metadata_db_connection_in_old_version(self, version):
         values = {
             "data": {"metadataConnection": {**self.non_chart_database_values}},
         }
-        connection = self._get_connection(values)
-
-        assert "db+postgresql://someuser:somepass@somehost:7777/somedb?sslmode=allow" == connection
+        connection = self._get_connection(self._get_values_with_version(values=values, version=version))
+        self._assert_for_old_version(
+            version,
+            value=connection,
+            expected_value="db+postgresql://someuser:somepass@somehost:7777/somedb?sslmode=allow",
+        )
 
     def test_should_correctly_use_non_chart_database(self):
         values = {"data": {"resultBackendConnection": {**self.non_chart_database_values}}}