Repository: apache/airflow

Issue:
API Endpoints - /xcomEntries/{xcom_key} cannot deserialize customized xcom backend
### Description

We use S3 as our xcom backend database and write serialize/deserialize method for xcoms.
However, when we want to access xcom through REST API, it returns the s3 file url instead of the deserialized value. Could you please add the feature to support customized xcom backend for REST API access?

### Use case/motivation

_No response_

### Related issues

_No response_

### Are you willing to submit a PR?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)



Discussion:
Thanks for opening your first issue here! Be sure to follow the issue template!

Can you provide the traceback emitted in the logs when the error occurs?
There is no error. So what we did is saving the json into a pickle file by overwriting the serialize_value and deserialize_value method. And we we do ti.xcom_pull we will get the actual value we saved.
But now when we get xcom through REST API, we will get the pickle file url like in the airflow UI
![image](https://user-images.githubusercontent.com/5023850/188605045-f0541599-0765-4c47-9a98-d3d77f88ea32.png)
And we check the API code and it seems it did not call deserialize_value to get the actual value we want.
![image](https://user-images.githubusercontent.com/5023850/188603661-ee3bb40a-5fc8-4d07-afd0-f8e99b7ae173.png)

Ah I see, thanks for the additional context. I think calling `deseralize_value` would be problematic because not everything deserialised can be sent through the wire, which is probably why the current implementation doesn’t do that.

One way I can think of to make this work would be to add an additional function in the XCom backend protocol for “serialise for API”. The default (JSON) backend would simply use the same serialisation logic used to save the value to the database, but custom backends can implement it differently to dump data.
Cool. That's what we want with the API. We look forward to having this feature in 2.4. Thanks in advance!
Note that we’re quite close to the feature freeze now (it should happen sometime this month), so I can’t guarantee this will make it into 2.4.
I see. We will do some workaround at this moment. No worries. Thank you.

Resolving patch:
diff --git a/airflow/api_connexion/endpoints/xcom_endpoint.py b/airflow/api_connexion/endpoints/xcom_endpoint.py
--- a/airflow/api_connexion/endpoints/xcom_endpoint.py
+++ b/airflow/api_connexion/endpoints/xcom_endpoint.py
@@ -14,6 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import copy
 from typing import Optional
 
 from flask import g
@@ -68,7 +69,7 @@ def get_xcom_entries(
     query = query.order_by(DR.execution_date, XCom.task_id, XCom.dag_id, XCom.key)
     total_entries = query.count()
     query = query.offset(offset).limit(limit)
-    return xcom_collection_schema.dump(XComCollection(xcom_entries=query.all(), total_entries=total_entries))
+    return xcom_collection_schema.dump(XComCollection(xcom_entries=query, total_entries=total_entries))
 
 
 @security.requires_access(
@@ -86,14 +87,28 @@ def get_xcom_entry(
     task_id: str,
     dag_run_id: str,
     xcom_key: str,
+    deserialize: bool = False,
     session: Session = NEW_SESSION,
 ) -> APIResponse:
     """Get an XCom entry"""
-    query = session.query(XCom).filter(XCom.dag_id == dag_id, XCom.task_id == task_id, XCom.key == xcom_key)
+    if deserialize:
+        query = session.query(XCom, XCom.value)
+    else:
+        query = session.query(XCom)
+
+    query = query.filter(XCom.dag_id == dag_id, XCom.task_id == task_id, XCom.key == xcom_key)
     query = query.join(DR, and_(XCom.dag_id == DR.dag_id, XCom.run_id == DR.run_id))
     query = query.filter(DR.run_id == dag_run_id)
 
-    query_object = query.one_or_none()
-    if not query_object:
+    item = query.one_or_none()
+    if item is None:
         raise NotFound("XCom entry not found")
-    return xcom_schema.dump(query_object)
+
+    if deserialize:
+        xcom, value = item
+        stub = copy.copy(xcom)
+        stub.value = value
+        stub.value = XCom.deserialize_value(stub)
+        item = stub
+
+    return xcom_schema.dump(item)


Test patch:
diff --git a/tests/api_connexion/endpoints/test_xcom_endpoint.py b/tests/api_connexion/endpoints/test_xcom_endpoint.py
--- a/tests/api_connexion/endpoints/test_xcom_endpoint.py
+++ b/tests/api_connexion/endpoints/test_xcom_endpoint.py
@@ -15,20 +15,34 @@
 # specific language governing permissions and limitations
 # under the License.
 from datetime import timedelta
+from unittest import mock
 
 import pytest
-from parameterized import parameterized
 
-from airflow.models import DagModel, DagRun, TaskInstance, XCom
+from airflow.models.dag import DagModel
+from airflow.models.dagrun import DagRun
+from airflow.models.taskinstance import TaskInstance
+from airflow.models.xcom import BaseXCom, XCom, resolve_xcom_backend
 from airflow.operators.empty import EmptyOperator
 from airflow.security import permissions
 from airflow.utils.dates import parse_execution_date
 from airflow.utils.session import create_session
+from airflow.utils.timezone import utcnow
 from airflow.utils.types import DagRunType
 from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_user
+from tests.test_utils.config import conf_vars
 from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_db_xcom
 
 
+class CustomXCom(BaseXCom):
+    @classmethod
+    def deserialize_value(cls, xcom: XCom):
+        return f"real deserialized {super().deserialize_value(xcom)}"
+
+    def orm_deserialize_value(self):
+        return f"orm deserialized {super().orm_deserialize_value()}"
+
+
 @pytest.fixture(scope="module")
 def configured_app(minimal_app_for_api):
     app = minimal_app_for_api
@@ -145,7 +159,7 @@ def test_should_raise_403_forbidden(self):
         )
         assert response.status_code == 403
 
-    def _create_xcom_entry(self, dag_id, run_id, execution_date, task_id, xcom_key):
+    def _create_xcom_entry(self, dag_id, run_id, execution_date, task_id, xcom_key, *, backend=XCom):
         with create_session() as session:
             dagrun = DagRun(
                 dag_id=dag_id,
@@ -158,7 +172,7 @@ def _create_xcom_entry(self, dag_id, run_id, execution_date, task_id, xcom_key):
             ti = TaskInstance(EmptyOperator(task_id=task_id), run_id=run_id)
             ti.dag_id = dag_id
             session.add(ti)
-        XCom.set(
+        backend.set(
             key=xcom_key,
             value="TEST_VALUE",
             run_id=run_id,
@@ -166,6 +180,26 @@ def _create_xcom_entry(self, dag_id, run_id, execution_date, task_id, xcom_key):
             dag_id=dag_id,
         )
 
+    @pytest.mark.parametrize(
+        "query, expected_value",
+        [
+            pytest.param("?deserialize=true", "real deserialized TEST_VALUE", id="true"),
+            pytest.param("?deserialize=false", "orm deserialized TEST_VALUE", id="false"),
+            pytest.param("", "orm deserialized TEST_VALUE", id="default"),
+        ],
+    )
+    @conf_vars({("core", "xcom_backend"): "tests.api_connexion.endpoints.test_xcom_endpoint.CustomXCom"})
+    def test_custom_xcom_deserialize(self, query, expected_value):
+        XCom = resolve_xcom_backend()
+        self._create_xcom_entry("dag", "run", utcnow(), "task", "key", backend=XCom)
+
+        url = f"/api/v1/dags/dag/dagRuns/run/taskInstances/task/xcomEntries/key{query}"
+        with mock.patch("airflow.api_connexion.endpoints.xcom_endpoint.XCom", XCom):
+            response = self.client.get(url, environ_overrides={'REMOTE_USER': "test"})
+
+        assert response.status_code == 200
+        assert response.json["value"] == expected_value
+
 
 class TestGetXComEntries(TestXComEndpoint):
     def test_should_respond_200(self):
@@ -386,7 +420,8 @@ def setup_method(self):
         self.execution_date_parsed = parse_execution_date(self.execution_date)
         self.run_id = DagRun.generate_run_id(DagRunType.MANUAL, self.execution_date_parsed)
 
-    @parameterized.expand(
+    @pytest.mark.parametrize(
+        "query_params, expected_xcom_ids",
         [
             (
                 "limit=1",
@@ -433,12 +468,12 @@ def setup_method(self):
                 "limit=2&offset=2",
                 ["TEST_XCOM_KEY2", "TEST_XCOM_KEY3"],
             ),
-        ]
+        ],
     )
     def test_handle_limit_offset(self, query_params, expected_xcom_ids):
-        url = "/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id}/xcomEntries?{query_params}"
-        url = url.format(
-            dag_id=self.dag_id, dag_run_id=self.run_id, task_id=self.task_id, query_params=query_params
+        url = (
+            f"/api/v1/dags/{self.dag_id}/dagRuns/{self.run_id}/taskInstances/{self.task_id}/xcomEntries"
+            f"?{query_params}"
         )
         with create_session() as session:
             dagrun = DagRun(