Repository: PrefectHQ/prefect

Issue:
Parameter 'priority' is ignored when creating work queues
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

I wrote a function to create 3 work queues with specified priority (10, 20 and 30). Work queues are created but priority is the default one (2,3,4).

![Screenshot from 2023-05-31 13-46-29](https://github.com/PrefectHQ/prefect/assets/4247916/2600f273-cffe-428f-8cd6-6653c913b88b)


### Reproduction

```python3
async def create_work_queues():
    async with get_client() as client:
        print("Creating work queues ...")
        queues = await client.read_work_queues("default-agent-pool")
        names = {queue.name for queue in queues}
        print(f"Existing work queues: {names}")
        tasks = []
        if "high" not in names:
            tasks.append(client.create_work_queue("high", priority=10))
        if "medium" not in names:
            tasks.append(client.create_work_queue("medium", priority=20))
        if "low" not in names:
            tasks.append(client.create_work_queue("low", priority=30))

        await asyncio.gather(*tasks)
        print("Finished creating work queues")
```


### Error

_No response_

### Versions

```Text
Version:             2.10.11
API version:         0.8.4
Python version:      3.11.3
Git commit:          8c651ffc
Built:               Thu, May 25, 2023 2:59 PM
OS/Arch:             linux/x86_64
Profile:             default
Server type:         server
```


### Additional context

_No response_


Discussion:

Resolving patch:
diff --git a/src/prefect/server/models/work_queues.py b/src/prefect/server/models/work_queues.py
--- a/src/prefect/server/models/work_queues.py
+++ b/src/prefect/server/models/work_queues.py
@@ -16,7 +16,10 @@
 from prefect.server.database.dependencies import inject_db
 from prefect.server.database.interface import PrefectDBInterface
 from prefect.server.exceptions import ObjectNotFoundError
-from prefect.server.models.workers import DEFAULT_AGENT_WORK_POOL_NAME
+from prefect.server.models.workers import (
+    DEFAULT_AGENT_WORK_POOL_NAME,
+    bulk_update_work_queue_priorities,
+)
 from prefect.server.schemas.states import StateType
 
 
@@ -39,7 +42,7 @@ async def create_work_queue(
         db.WorkQueue: the newly-created or updated WorkQueue
 
     """
-    data = work_queue.dict(exclude={"priority"})
+    data = work_queue.dict()
 
     if data.get("work_pool_id") is None:
         # If no work pool is provided, get or create the default agent work pool
@@ -66,16 +69,41 @@ async def create_work_queue(
 
     # Set the priority to be the max priority + 1
     # This will make the new queue the lowest priority
-    max_priority_query = sa.select(
-        sa.func.coalesce(sa.func.max(db.WorkQueue.priority), 0)
-    ).where(db.WorkQueue.work_pool_id == data["work_pool_id"])
-    priority = (await session.execute(max_priority_query)).scalar()
+    if data["priority"] is None:
+        # Set the priority to be the first priority value that isn't already taken
+        priorities_query = sa.select(db.WorkQueue.priority).where(
+            db.WorkQueue.work_pool_id == data["work_pool_id"]
+        )
+        priorities = (await session.execute(priorities_query)).scalars().all()
+
+        priority = None
+        for i, p in enumerate(sorted(priorities)):
+            # if a rank was skipped (e.g. the set priority is different than the
+            # enumerated priority) then we can "take" that spot for this work
+            # queue
+            if i + 1 != p:
+                priority = i + 1
+                break
+
+        # otherwise take the maximum priority plus one
+        if priority is None:
+            priority = max(priorities, default=0) + 1
+
+        data["priority"] = priority
 
-    model = db.WorkQueue(**data, priority=priority + 1)
+    model = db.WorkQueue(**data)
 
     session.add(model)
     await session.flush()
 
+    if work_queue.priority:
+        await bulk_update_work_queue_priorities(
+            session=session,
+            work_pool_id=data["work_pool_id"],
+            new_priorities={model.id: work_queue.priority},
+            db=db,
+        )
+
     return model
 
 
diff --git a/src/prefect/server/models/workers.py b/src/prefect/server/models/workers.py
--- a/src/prefect/server/models/workers.py
+++ b/src/prefect/server/models/workers.py
@@ -259,18 +259,30 @@ async def create_work_queue(
         db.WorkQueue: the newly-created WorkQueue
 
     """
+    data = work_queue.dict(exclude={"work_pool_id"})
+    if work_queue.priority is None:
+        # Set the priority to be the first priority value that isn't already taken
+        priorities_query = sa.select(db.WorkQueue.priority).where(
+            db.WorkQueue.work_pool_id == work_pool_id
+        )
+        priorities = (await session.execute(priorities_query)).scalars().all()
 
-    max_priority_query = sa.select(
-        sa.func.coalesce(sa.func.max(db.WorkQueue.priority), 0)
-    ).where(db.WorkQueue.work_pool_id == work_pool_id)
-    priority = (await session.execute(max_priority_query)).scalar()
+        priority = None
+        for i, p in enumerate(sorted(priorities)):
+            # if a rank was skipped (e.g. the set priority is different than the
+            # enumerated priority) then we can "take" that spot for this work
+            # queue
+            if i + 1 != p:
+                priority = i + 1
+                break
 
-    model = db.WorkQueue(
-        **work_queue.dict(exclude={"priority", "work_pool_id"}),
-        work_pool_id=work_pool_id,
-        # initialize the priority as the current max priority + 1
-        priority=priority + 1
-    )
+        # otherwise take the maximum priority plus one
+        if priority is None:
+            priority = max(priorities, default=0) + 1
+
+        data["priority"] = priority
+
+    model = db.WorkQueue(**data, work_pool_id=work_pool_id)
 
     session.add(model)
     await session.flush()
@@ -293,11 +305,16 @@ async def bulk_update_work_queue_priorities(
     db: PrefectDBInterface,
 ):
     """
-    This is a brute force update of all work pool queue priorities for a given worker
+    This is a brute force update of all work pool queue priorities for a given work
     pool.
 
     It loads all queues fully into memory, sorts them, and flushes the update to
-    the db.
+    the db. The algorithm ensures that priorities are unique integers > 0, and
+    makes the minimum number of changes required to satisfy the provided
+    `new_priorities`. For example, if no queues currently have the provided
+    `new_priorities`, then they are assigned without affecting other queues. If
+    they are held by other queues, then those queues' priorities are
+    incremented as necessary.
 
     Updating queue priorities is not a common operation (happens on the same scale as
     queue modification, which is significantly less than reading from queues),
@@ -308,6 +325,7 @@ async def bulk_update_work_queue_priorities(
     if len(set(new_priorities.values())) != len(new_priorities):
         raise ValueError("Duplicate target priorities provided")
 
+    # get all the work queues, sorted by priority
     work_queues_query = (
         sa.select(db.WorkQueue)
         .where(db.WorkQueue.work_pool_id == work_pool_id)
@@ -316,14 +334,31 @@ async def bulk_update_work_queue_priorities(
     result = await session.execute(work_queues_query)
     all_work_queues = result.scalars().all()
 
+    # split the queues into those that need to be updated and those that don't
     work_queues = [wq for wq in all_work_queues if wq.id not in new_priorities]
     updated_queues = [wq for wq in all_work_queues if wq.id in new_priorities]
 
+    # update queue priorities and insert them into the appropriate place in the
+    # full list of queues
     for queue in sorted(updated_queues, key=lambda wq: new_priorities[wq.id]):
-        work_queues.insert(new_priorities[queue.id] - 1, queue)
-
-    for i, queue in enumerate(work_queues):
-        queue.priority = i + 1
+        queue.priority = new_priorities[queue.id]
+        for i, wq in enumerate(work_queues):
+            if wq.priority >= new_priorities[queue.id]:
+                work_queues.insert(i, queue)
+                break
+
+    # walk through the queues and update their priorities such that the
+    # priorities are sequential. Do this by tracking that last priority seen and
+    # ensuring that each successive queue's priority is higher than it. This
+    # will maintain queue order and ensure increasing priorities with minimal
+    # changes.
+    last_priority = 0
+    for queue in work_queues:
+        if queue.priority <= last_priority:
+            last_priority += 1
+            queue.priority = last_priority
+        else:
+            last_priority = queue.priority
 
     await session.flush()
 


Test patch:
diff --git a/tests/server/api/test_work_queues.py b/tests/server/api/test_work_queues.py
--- a/tests/server/api/test_work_queues.py
+++ b/tests/server/api/test_work_queues.py
@@ -32,6 +32,58 @@ async def test_create_work_queue(
         assert str(work_queue.id) == work_queue_id
         assert work_queue.name == "wq-1"
 
+    async def test_create_work_queue_with_priority(
+        self,
+        client,
+        session,
+        work_pool,
+    ):
+        data = dict(name="my-wpq", priority=99)
+        response = await client.post(
+            "/work_queues/",
+            json=data,
+        )
+        assert response.status_code == 201
+        assert response.json()["priority"] == 99
+        work_queue_id = response.json()["id"]
+
+        work_queue = await models.work_queues.read_work_queue(
+            session=session, work_queue_id=work_queue_id
+        )
+        assert work_queue.priority == 99
+
+    async def test_create_work_queue_with_no_priority_when_low_priority_set(
+        self,
+        client,
+        work_pool,
+    ):
+        response = await client.post("/work_queues/", json=dict(name="wpq-1"))
+        # priority 2 because the default queue exists
+        assert response.json()["priority"] == 2
+
+        response2 = await client.post("/work_queues/", json=dict(name="wpq-2"))
+        assert response2.json()["priority"] == 3
+
+    async def test_create_work_queue_with_no_priority_when_high_priority_set(
+        self,
+        client,
+        session,
+        work_pool,
+    ):
+        response = await client.post(
+            "/work_queues/", json=dict(name="wpq-1", priority=99)
+        )
+        assert response.json()["priority"] == 99
+        work_queue_id = response.json()["id"]
+
+        response2 = await client.post("/work_queues/", json=dict(name="wpq-2"))
+        assert response2.json()["priority"] == 2
+
+        work_queue = await models.work_queues.read_work_queue(
+            session=session, work_queue_id=work_queue_id
+        )
+        assert work_queue.priority == 99
+
     async def test_create_work_queue_raises_error_on_existing_name(
         self, client, work_queue
     ):
diff --git a/tests/server/api/test_workers.py b/tests/server/api/test_workers.py
--- a/tests/server/api/test_workers.py
+++ b/tests/server/api/test_workers.py
@@ -471,6 +471,64 @@ async def test_create_work_queue(self, client, work_pool):
         assert result.name == "test-queue"
         assert result.description == "test queue"
 
+    async def test_create_work_queue_with_priority(
+        self,
+        client,
+        session,
+        work_pool,
+    ):
+        data = dict(name="my-wpq", priority=99)
+        response = await client.post(
+            f"work_pools/{work_pool.name}/queues",
+            json=data,
+        )
+        assert response.status_code == 201
+        assert response.json()["priority"] == 99
+        work_queue_id = response.json()["id"]
+
+        work_queue = await models.workers.read_work_queue(
+            session=session, work_queue_id=work_queue_id
+        )
+        assert work_queue.priority == 99
+
+    async def test_create_work_queue_with_no_priority_when_low_priority_set(
+        self,
+        client,
+        work_pool,
+    ):
+        response = await client.post(
+            f"work_pools/{work_pool.name}/queues", json=dict(name="wpq-1")
+        )
+        # priority 2 because the default queue exists
+        assert response.json()["priority"] == 2
+
+        response2 = await client.post(
+            f"work_pools/{work_pool.name}/queues", json=dict(name="wpq-2")
+        )
+        assert response2.json()["priority"] == 3
+
+    async def test_create_work_queue_with_no_priority_when_high_priority_set(
+        self,
+        client,
+        session,
+        work_pool,
+    ):
+        response = await client.post(
+            f"work_pools/{work_pool.name}/queues", json=dict(name="wpq-1", priority=99)
+        )
+        assert response.json()["priority"] == 99
+        work_queue_id = response.json()["id"]
+
+        response2 = await client.post(
+            f"work_pools/{work_pool.name}/queues", json=dict(name="wpq-2")
+        )
+        assert response2.json()["priority"] == 2
+
+        work_queue = await models.workers.read_work_queue(
+            session=session, work_queue_id=work_queue_id
+        )
+        assert work_queue.priority == 99
+
 
 class TestReadWorkQueue:
     async def test_read_work_queue(self, client, work_pool):
diff --git a/tests/server/models/test_workers.py b/tests/server/models/test_workers.py
--- a/tests/server/models/test_workers.py
+++ b/tests/server/models/test_workers.py
@@ -336,8 +336,8 @@ async def test_read_work_queues_sorts_by_priority(self, session, work_pool):
         assert len(result) == 4
         assert (result[0].name, result[0].priority) == ("default", 1)
         assert (result[1].name, result[1].priority) == ("C", 2)
-        assert (result[2].name, result[2].priority) == ("B", 3)
-        assert (result[3].name, result[3].priority) == ("A", 4)
+        assert (result[2].name, result[2].priority) == ("B", 4)
+        assert (result[3].name, result[3].priority) == ("A", 100)
 
 
 class TestUpdateWorkQueue:
@@ -375,19 +375,6 @@ async def test_update_work_queue_zero_concurrency(self, session, work_queue):
         )
         assert result.concurrency_limit == 0
 
-    async def test_update_work_queue_priority_is_normalized_for_number_of_queues(
-        self, session, work_queue_1
-    ):
-        assert await models.workers.update_work_queue(
-            session=session,
-            work_queue_id=work_queue_1.id,
-            work_queue=schemas.actions.WorkQueueUpdate(priority=100),
-        )
-        result = await models.workers.read_work_queue(
-            session=session, work_queue_id=work_queue_1.id
-        )
-        assert result.priority == 2
-
 
 class TestUpdateWorkQueuePriorities:
     @pytest.fixture(autouse=True)
@@ -517,8 +504,8 @@ async def test_update_priorities_with_empty_new_priority_to_recompute(
         assert {q.name: q.priority for q in all_queues} == {
             "A": 1,
             "B": 2,
-            "D": 3,
-            "E": 4,
+            "D": 4,
+            "E": 5,
         }
 
 
@@ -569,7 +556,7 @@ async def test_delete_queue_updates_priorities(self, session, work_pool):
         assert len(result) == 3
         assert (result[0].name, result[0].priority) == ("default", 1)
         assert (result[1].name, result[1].priority) == ("A", 2)
-        assert (result[2].name, result[2].priority) == ("C", 3)
+        assert (result[2].name, result[2].priority) == ("C", 4)
 
 
 class TestWorkerHeartbeat: