Repository: apache/airflow

Issue:
API Endpoint - CRUD - Pool
Hello 

We need to create several endpoints that perform basic CRUD operations on **Pools**. We need the following endpoints:

POST /pools
DELETE /pools/{pool_name}
PATCH /pools/{pool_name}

Detailed information is available in the issue:
https://github.com/apache/airflow/issues/8118

LOVE,


Discussion:
Hi @mik-laj can you assign me this issue please.
Hi @OmairK I assigned you to this task.

Resolving patch:
diff --git a/airflow/api_connexion/endpoints/pool_endpoint.py b/airflow/api_connexion/endpoints/pool_endpoint.py
--- a/airflow/api_connexion/endpoints/pool_endpoint.py
+++ b/airflow/api_connexion/endpoints/pool_endpoint.py
@@ -14,21 +14,29 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from flask import request
+from flask import Response, request
+from marshmallow import ValidationError
 from sqlalchemy import func
+from sqlalchemy.exc import IntegrityError
 
 from airflow.api_connexion import parameters
-from airflow.api_connexion.exceptions import NotFound
+from airflow.api_connexion.exceptions import AlreadyExists, BadRequest, NotFound
 from airflow.api_connexion.schemas.pool_schema import PoolCollection, pool_collection_schema, pool_schema
 from airflow.models.pool import Pool
 from airflow.utils.session import provide_session
 
 
-def delete_pool():
+@provide_session
+def delete_pool(pool_name: str, session):
     """
     Delete a pool
     """
-    raise NotImplementedError("Not implemented yet.")
+    if pool_name == "default_pool":
+        raise BadRequest(detail="Default Pool can't be deleted")
+    elif session.query(Pool).filter(Pool.pool == pool_name).delete() == 0:
+        raise NotFound(detail=f"Pool with name:'{pool_name}' not found")
+    else:
+        return Response(status=204)
 
 
 @provide_session
@@ -37,9 +45,8 @@ def get_pool(pool_name, session):
     Get a pool
     """
     obj = session.query(Pool).filter(Pool.pool == pool_name).one_or_none()
-
     if obj is None:
-        raise NotFound("Pool not found")
+        raise NotFound(detail=f"Pool with name:'{pool_name}' not found")
     return pool_schema.dump(obj)
 
 
@@ -53,18 +60,80 @@ def get_pools(session):
 
     total_entries = session.query(func.count(Pool.id)).scalar()
     pools = session.query(Pool).order_by(Pool.id).offset(offset).limit(limit).all()
-    return pool_collection_schema.dump(PoolCollection(pools=pools, total_entries=total_entries)).data
+    return pool_collection_schema.dump(
+        PoolCollection(pools=pools, total_entries=total_entries)
+    ).data
 
 
-def patch_pool():
+@provide_session
+def patch_pool(pool_name, session, update_mask=None):
     """
     Update a pool
     """
-    raise NotImplementedError("Not implemented yet.")
+    # Only slots can be modified in 'default_pool'
+    try:
+        if pool_name == Pool.DEFAULT_POOL_NAME and request.json["name"] != Pool.DEFAULT_POOL_NAME:
+            if update_mask and len(update_mask) == 1 and update_mask[0].strip() == "slots":
+                pass
+            else:
+                raise BadRequest(detail="Default Pool's name can't be modified")
+    except KeyError:
+        pass
+
+    pool = session.query(Pool).filter(Pool.pool == pool_name).first()
+    if not pool:
+        raise NotFound(detail=f"Pool with name:'{pool_name}' not found")
+
+    try:
+        patch_body = pool_schema.load(request.json).data
+    except ValidationError as err:
+        raise BadRequest(detail=err.messages.get("_schema", [err.messages])[0])
+
+    if update_mask:
+        update_mask = [i.strip() for i in update_mask]
+        _patch_body = {}
+        try:
+            update_mask = [
+                pool_schema.declared_fields[field].attribute
+                if pool_schema.declared_fields[field].attribute
+                else field
+                for field in update_mask
+            ]
+        except KeyError as err:
+            raise BadRequest(detail=f"Invalid field: {err.args[0]} in update mask")
+        _patch_body = {field: patch_body[field] for field in update_mask}
+        patch_body = _patch_body
 
+    else:
+        for field in ["name", "slots"]:
+            if field not in request.json.keys():
+                raise BadRequest(detail=f"'{field}' is a required property")
 
-def post_pool():
+    for key, value in patch_body.items():
+        setattr(pool, key, value)
+    session.commit()
+    return pool_schema.dump(pool)
+
+
+@provide_session
+def post_pool(session):
     """
-    Create aa pool
+    Create a pool
     """
-    raise NotImplementedError("Not implemented yet.")
+    required_fields = ["name", "slots"]  # Pool would require both fields in the post request
+    for field in required_fields:
+        if field not in request.json.keys():
+            raise BadRequest(detail=f"'{field}' is a required property")
+
+    try:
+        post_body = pool_schema.load(request.json, session=session).data
+    except ValidationError as err:
+        raise BadRequest(detail=err.messages.get("_schema", [err.messages])[0])
+
+    pool = Pool(**post_body)
+    try:
+        session.add(pool)
+        session.commit()
+        return pool_schema.dump(pool)
+    except IntegrityError:
+        raise AlreadyExists(detail=f"Pool: {post_body['pool']} already exists")
diff --git a/airflow/api_connexion/schemas/pool_schema.py b/airflow/api_connexion/schemas/pool_schema.py
--- a/airflow/api_connexion/schemas/pool_schema.py
+++ b/airflow/api_connexion/schemas/pool_schema.py
@@ -17,7 +17,7 @@
 
 from typing import List, NamedTuple
 
-from marshmallow import Schema, fields
+from marshmallow import Schema, ValidationError, fields, validates_schema
 from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
 
 from airflow.models.pool import Pool
@@ -30,7 +30,6 @@ class Meta:
         """Meta"""
 
         model = Pool
-        load_instance = True
         exclude = ("pool",)
 
     name = auto_field("pool")
@@ -68,6 +67,13 @@ def get_open_slots(obj: Pool) -> int:
         """
         return obj.open_slots()
 
+    @validates_schema(pass_original=True)
+    def check_unknown_fields(self, data, original_data):  # pylint: disable=unused-argument
+        """ Validates unknown field """
+        unknown = set(original_data) - set(self.fields)
+        if unknown:
+            raise ValidationError(f"Extra arguments passed: {list(unknown)}")
+
 
 class PoolCollection(NamedTuple):
     """List of Pools with metadata"""


Test patch:
diff --git a/tests/api_connexion/endpoints/test_pool_endpoint.py b/tests/api_connexion/endpoints/test_pool_endpoint.py
--- a/tests/api_connexion/endpoints/test_pool_endpoint.py
+++ b/tests/api_connexion/endpoints/test_pool_endpoint.py
@@ -134,9 +134,346 @@ def test_response_404(self):
         assert response.status_code == 404
         self.assertEqual(
             {
-                "detail": None,
+                "detail": "Pool with name:'invalid_pool' not found",
                 "status": 404,
-                "title": "Pool not found",
+                "title": "Object not found",
+                "type": "about:blank",
+            },
+            response.json,
+        )
+
+
+class TestDeletePool(TestBasePoolEndpoints):
+    @provide_session
+    def test_response_204(self, session):
+        pool_name = "test_pool"
+        pool_instance = Pool(pool=pool_name, slots=3)
+        session.add(pool_instance)
+        session.commit()
+
+        response = self.client.delete(f"api/v1/pools/{pool_name}")
+        assert response.status_code == 204
+        # Check if the pool is deleted from the db
+        response = self.client.get(f"api/v1/pools/{pool_name}")
+        self.assertEqual(response.status_code, 404)
+
+    def test_response_404(self):
+        response = self.client.delete("api/v1/pools/invalid_pool")
+        self.assertEqual(response.status_code, 404)
+        self.assertEqual(
+            {
+                "detail": "Pool with name:'invalid_pool' not found",
+                "status": 404,
+                "title": "Object not found",
+                "type": "about:blank",
+            },
+            response.json,
+        )
+
+
+class TestPostPool(TestBasePoolEndpoints):
+    def test_response_200(self):
+        response = self.client.post(
+            "api/v1/pools", json={"name": "test_pool_a", "slots": 3}
+        )
+        assert response.status_code == 200
+        self.assertEqual(
+            {
+                "name": "test_pool_a",
+                "slots": 3,
+                "occupied_slots": 0,
+                "running_slots": 0,
+                "queued_slots": 0,
+                "open_slots": 3,
+            },
+            response.json,
+        )
+
+    @provide_session
+    def test_response_409(self, session):
+        pool_name = "test_pool_a"
+        pool_instance = Pool(pool=pool_name, slots=3)
+        session.add(pool_instance)
+        session.commit()
+        response = self.client.post(
+            "api/v1/pools", json={"name": "test_pool_a", "slots": 3}
+        )
+        assert response.status_code == 409
+        self.assertEqual(
+            {
+                "detail": f"Pool: {pool_name} already exists",
+                "status": 409,
+                "title": "Object already exists",
+                "type": "about:blank",
+            },
+            response.json,
+        )
+
+    @parameterized.expand(
+        [
+            ("for missing pool name", {"slots": 3}, "'name' is a required property",),
+            (
+                "for missing slots",
+                {"name": "invalid_pool"},
+                "'slots' is a required property",
+            ),
+            (
+                "for extra fields",
+                {"name": "invalid_pool", "slots": 3, "extra_field_1": "extra"},
+                "Extra arguments passed: ['extra_field_1']",
+            ),
+        ]
+    )
+    def test_response_400(self, name, request_json, error_detail):
+        del name
+        response = self.client.post("api/v1/pools", json=request_json)
+        assert response.status_code == 400
+        self.assertDictEqual(
+            {
+                "detail": error_detail,
+                "status": 400,
+                "title": "Bad request",
+                "type": "about:blank",
+            },
+            response.json,
+        )
+
+
+class TestPatchPool(TestBasePoolEndpoints):
+    @provide_session
+    def test_response_200(self, session):
+        pool = Pool(pool="test_pool", slots=2)
+        session.add(pool)
+        session.commit()
+        response = self.client.patch(
+            "api/v1/pools/test_pool", json={"name": "test_pool_a", "slots": 3}
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(
+            {
+                "occupied_slots": 0,
+                "queued_slots": 0,
+                "name": "test_pool_a",
+                "open_slots": 3,
+                "running_slots": 0,
+                "slots": 3,
+            },
+            response.json,
+        )
+
+    @parameterized.expand(
+        [
+            # Missing properties
+            ("'name' is a required property", {"slots": 3}),
+            ("'slots' is a required property", {"name": "test_pool_a"}),
+            # Extra properties
+            (
+                "Extra arguments passed: ['extra_field']",
+                {"name": "test_pool_a", "slots": 3, "extra_field": "extra"},
+            ),
+        ]
+    )
+    @provide_session
+    def test_response_400(self, error_detail, request_json, session):
+        pool = Pool(pool="test_pool", slots=2)
+        session.add(pool)
+        session.commit()
+        response = self.client.patch("api/v1/pools/test_pool", json=request_json)
+        assert response.status_code == 400
+        self.assertEqual(
+            {
+                "detail": error_detail,
+                "status": 400,
+                "title": "Bad request",
+                "type": "about:blank",
+            },
+            response.json,
+        )
+
+
+class TestModifyDefaultPool(TestBasePoolEndpoints):
+    def test_delete_400(self):
+        response = self.client.delete("api/v1/pools/default_pool")
+        assert response.status_code == 400
+        self.assertEqual(
+            {
+                "detail": "Default Pool can't be deleted",
+                "status": 400,
+                "title": "Bad request",
+                "type": "about:blank",
+            },
+            response.json,
+        )
+
+    @parameterized.expand(
+        [
+            (
+                "400 No update mask",
+                400,
+                "api/v1/pools/default_pool",
+                {"name": "test_pool_a", "slots": 3},
+                {
+                    "detail": "Default Pool's name can't be modified",
+                    "status": 400,
+                    "title": "Bad request",
+                    "type": "about:blank",
+                },
+            ),
+            (
+                "400 Update mask with both fields",
+                400,
+                "api/v1/pools/default_pool?update_mask=name, slots",
+                {"name": "test_pool_a", "slots": 3},
+                {
+                    "detail": "Default Pool's name can't be modified",
+                    "status": 400,
+                    "title": "Bad request",
+                    "type": "about:blank",
+                },
+            ),
+            (
+                "200 Update mask with slots",
+                200,
+                "api/v1/pools/default_pool?update_mask=slots",
+                {"name": "test_pool_a", "slots": 3},
+                {
+                    "occupied_slots": 0,
+                    "queued_slots": 0,
+                    "name": "default_pool",
+                    "open_slots": 3,
+                    "running_slots": 0,
+                    "slots": 3,
+                },
+            ),
+            (
+                "200 Update mask with slots and name",
+                200,
+                "api/v1/pools/default_pool?update_mask=name,slots",
+                {"name": "default_pool", "slots": 3},
+                {
+                    "occupied_slots": 0,
+                    "queued_slots": 0,
+                    "name": "default_pool",
+                    "open_slots": 3,
+                    "running_slots": 0,
+                    "slots": 3,
+                },
+            ),
+            (
+                "200 no update mask",
+                200,
+                "api/v1/pools/default_pool",
+                {"name": "default_pool", "slots": 3},
+                {
+                    "occupied_slots": 0,
+                    "queued_slots": 0,
+                    "name": "default_pool",
+                    "open_slots": 3,
+                    "running_slots": 0,
+                    "slots": 3,
+                },
+            ),
+        ]
+    )
+    def test_patch(self, name, status_code, url, json, expected_response):
+        del name
+        response = self.client.patch(url, json=json)
+        assert response.status_code == status_code
+        self.assertEqual(response.json, expected_response)
+
+
+class TestPatchPoolWithUpdateMask(TestBasePoolEndpoints):
+    @parameterized.expand(
+        [
+            (
+                "api/v1/pools/test_pool?update_mask=name, slots",
+                {"name": "test_pool_a", "slots": 2},
+                "test_pool_a",
+                2,
+            ),
+            (
+                "api/v1/pools/test_pool?update_mask=name",
+                {"name": "test_pool_a", "slots": 2},
+                "test_pool_a",
+                3,
+            ),
+            (
+                "api/v1/pools/test_pool?update_mask=slots",
+                {"name": "test_pool_a", "slots": 2},
+                "test_pool",
+                2,
+            ),
+            (
+                "api/v1/pools/test_pool?update_mask=slots",
+                {"slots": 2},
+                "test_pool",
+                2,
+            ),
+        ]
+    )
+    @provide_session
+    def test_response_200(
+        self, url, patch_json, expected_name, expected_slots, session
+    ):
+        pool = Pool(pool="test_pool", slots=3)
+        session.add(pool)
+        session.commit()
+        response = self.client.patch(url, json=patch_json)
+        assert response.status_code == 200
+        self.assertEqual(
+            {
+                "name": expected_name,
+                "slots": expected_slots,
+                "occupied_slots": 0,
+                "running_slots": 0,
+                "queued_slots": 0,
+                "open_slots": expected_slots,
+            },
+            response.json,
+        )
+
+    @parameterized.expand(
+        [
+            (
+                "Patching read only field",
+                "Property is read-only - 'occupied_slots'",
+                "api/v1/pools/test_pool?update_mask=slots, name, occupied_slots",
+                {"name": "test_pool_a", "slots": 2, "occupied_slots": 1},
+            ),
+            (
+                "Patching read only field",
+                "Property is read-only - 'queued_slots'",
+                "api/v1/pools/test_pool?update_mask=slots, name, queued_slots",
+                {"name": "test_pool_a", "slots": 2, "queued_slots": 1},
+            ),
+            (
+                "Invalid update mask",
+                "'names' is not a valid Pool field",
+                "api/v1/pools/test_pool?update_mask=slots, names,",
+                {"name": "test_pool_a", "slots": 2},
+            ),
+            (
+                "Invalid update mask",
+                "'slot' is not a valid Pool field",
+                "api/v1/pools/test_pool?update_mask=slot, name,",
+                {"name": "test_pool_a", "slots": 2},
+            ),
+        ]
+    )
+    @provide_session
+    def test_response_400(self, name, error_detail, url, patch_json, session):
+        del name
+        pool = Pool(pool="test_pool", slots=3)
+        session.add(pool)
+        session.commit()
+        response = self.client.patch(url, json=patch_json)
+        assert response.status_code == 400
+        self.assertEqual
+        (
+            {
+                "detail": error_detail,
+                "status": 400,
+                "title": "Bad Request",
                 "type": "about:blank",
             },
             response.json,
diff --git a/tests/api_connexion/schemas/test_pool_schemas.py b/tests/api_connexion/schemas/test_pool_schemas.py
--- a/tests/api_connexion/schemas/test_pool_schemas.py
+++ b/tests/api_connexion/schemas/test_pool_schemas.py
@@ -53,7 +53,7 @@ def test_serialize(self, session):
     def test_desearialize(self, session):
         pool_dict = {"name": "test_pool", "slots": 3}
         deserialized_pool = pool_schema.load(pool_dict, session=session)
-        self.assertIsInstance(deserialized_pool.data, Pool)
+        self.assertNotIsInstance(deserialized_pool.data, Pool)  # Checks if load_instance is set to True
 
 
 class TestPoolCollectionSchema(unittest.TestCase):