Repository: PrefectHQ/prefect
Issue category: Feature Request

Issue:
Configure `ipc_mode` or `shm_size` on container creation
For some flows that utilize `multiprocessing` for ML workloads, I need to run containers with increased shared memory available under `/dev/shm` (`--shm_size`) or use the host's namespace (--ipc_mode='host').  Looking through the code, I believe this would expand `DockerWorkerJobConfiguration` to include these two settings, which are available in `Container.create`.  

Proposed additions:
- Add `ipc_mode` and `shm_size` to `DockerWorkerJobConfiguration`, both defaulting to `None` to preserve default behavior.

@desertaxle 


Hints:
Thanks for the issue @geoffrp! I think this is a great opportunity to expose more configurability for the Docker worker so that we don't need to update the code each time a new parameter is needed. 
I would tend to agree, lots of arguments to play with for custom containers.  Are you suggesting removing `DockerWorkerJobConfiguration`?  I'm assuming that this construct allows the UI to know what fields may be specified if not creating through the command line?
We'll need to expand `DockerWorkerJobConfiguration` to include a field that accepts an arbitrary dictionary of attributes passed directly to `docker-py` functions. This will allow users to modify the base job template of their work pools to suit their needs without requiring additional code changes to the `DockerWorker`.
@desertaxle Is there a prospective timeline for this feature?  I would be willing to assist if possible

Patch:
diff --git a/src/integrations/prefect-docker/prefect_docker/worker.py b/src/integrations/prefect-docker/prefect_docker/worker.py
index bd85f4525325..9ebba4986f16 100644
--- a/src/integrations/prefect-docker/prefect_docker/worker.py
+++ b/src/integrations/prefect-docker/prefect_docker/worker.py
@@ -101,6 +101,7 @@ class DockerWorkerJobConfiguration(BaseJobConfiguration):
             `mem_limit` is 300m and `memswap_limit` is not set, containers can use
             600m in total of memory and swap.
         privileged: Give extended privileges to created containers.
+        container_create_kwargs: Extra args for docker py when creating container.
     """
 
     image: str = Field(
@@ -165,11 +166,17 @@ class DockerWorkerJobConfiguration(BaseJobConfiguration):
             "600m in total of memory and swap."
         ),
     )
-
     privileged: bool = Field(
         default=False,
         description="Give extended privileges to created container.",
     )
+    container_create_kwargs: Optional[Dict[str, Any]] = Field(
+        default=None,
+        title="Container Configuration",
+        description=(
+            "Configuration for containers created by workers. See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/containers.html) for accepted values."
+        ),
+    )
 
     @validator("volumes")
     def _validate_volume_format(cls, volumes):
@@ -526,6 +533,18 @@ def _build_container_settings(
     ) -> Dict:
         """Builds a dictionary of container settings to pass to the Docker API."""
         network_mode = configuration.get_network_mode()
+
+        container_create_kwargs = (
+            configuration.container_create_kwargs
+            if configuration.container_create_kwargs
+            else {}
+        )
+        container_create_kwargs = {
+            k: v
+            for k, v in container_create_kwargs.items()
+            if k not in configuration.__fields__
+        }
+
         return dict(
             image=configuration.image,
             network=configuration.networks[0] if configuration.networks else None,
@@ -540,6 +559,7 @@ def _build_container_settings(
             mem_limit=configuration.mem_limit,
             memswap_limit=configuration.memswap_limit,
             privileged=configuration.privileged,
+            **container_create_kwargs,
         )
 
     def _create_and_start_container(


Test patch:
diff --git a/src/integrations/prefect-docker/tests/test_worker.py b/src/integrations/prefect-docker/tests/test_worker.py
index 1fa081ccd3b4..55130027f4af 100644
--- a/src/integrations/prefect-docker/tests/test_worker.py
+++ b/src/integrations/prefect-docker/tests/test_worker.py
@@ -850,6 +850,37 @@ async def test_task_infra_pid_includes_host_and_container_id(
     assert result.identifier == f"{FAKE_BASE_URL}:{FAKE_CONTAINER_ID}"
 
 
+async def test_container_create_kwargs(
+    mock_docker_client, flow_run, default_docker_worker_job_configuration
+):
+    default_docker_worker_job_configuration.container_create_kwargs = {
+        "hostname": "custom_name"
+    }
+    async with DockerWorker(work_pool_name="test") as worker:
+        await worker.run(
+            flow_run=flow_run, configuration=default_docker_worker_job_configuration
+        )
+    mock_docker_client.containers.create.assert_called_once()
+    hostname = mock_docker_client.containers.create.call_args[1].get("hostname")
+    assert hostname == "custom_name"
+
+
+async def test_container_create_kwargs_excludes_job_variables(
+    mock_docker_client, flow_run, default_docker_worker_job_configuration
+):
+    default_docker_worker_job_configuration.name = "job_config_name"
+    default_docker_worker_job_configuration.container_create_kwargs = {
+        "name": "create_kwarg_name"
+    }
+    async with DockerWorker(work_pool_name="test") as worker:
+        await worker.run(
+            flow_run=flow_run, configuration=default_docker_worker_job_configuration
+        )
+    mock_docker_client.containers.create.assert_called_once()
+    name = mock_docker_client.containers.create.call_args[1].get("name")
+    assert name == "job_config_name"
+
+
 async def test_task_status_receives_result_identifier(
     mock_docker_client, flow_run, default_docker_worker_job_configuration
 ):