Repository: PrefectHQ/prefect

Issue:
`DockerRegistry` block throws `ImageNotFound: 404 Client Error` despite correct authentication to private container registry
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

It doesn't seem like an issue introduced in the latest release only as I was able to reproduce the error on 2.2 and 2.1 too. 

**Problem**: authentication with private Docker registries doesn't seem to work - reproduced with a private DockerHub registry

### Reproduction
Block creation: https://github.com/anna-geller/prefect-docker-deployment/blob/main/blocks/dockerhub_block.py

Deployment:
```python
python blocks/dockerhub_block.py
prefect deployment build flows/healthcheck.py:healthcheck -n prod -q prod -ib docker-container/prod --apply
prefect deployment run healthcheck/prod
```


### Error

_No response_

### Versions

```
Version:             2.3.0
API version:         0.8.0
Python version:      3.9.12
Git commit:          8d9316c0
Built:               Tue, Aug 30, 2022 5:30 PM
OS/Arch:             darwin/arm64
Profile:             default
Server type:         hosted
```


### Additional context

might be fixed in https://github.com/PrefectHQ/prefect/pull/6622 


Discussion:
I introduced this issue on [SLACK ](https://prefect-community.slack.com/archives/CL09KU1K7/p1661945482939029)so I just want to add my agent log which prints out a a internal server error ("no basic auth credentials").

```
Starting v2.3.0 agent with ephemeral API...

  ___ ___ ___ ___ ___ ___ _____     _   ___ ___ _  _ _____
 | _ \ _ \ __| __| __/ __|_   _|   /_\ / __| __| \| |_   _|
 |  _/   / _|| _|| _| (__  | |    / _ \ (_ | _|| .` | | |
 |_| |_|_\___|_| |___\___| |_|   /_/ \_\___|___|_|\_| |_|


Agent started! Looking for work from queue(s): default...
15:24:30.641 | INFO    | prefect.agent - Submitting flow run '6acdbe6c-56f7-49fa-bf61-792d0f1d3df9'
15:24:31.795 | INFO    | prefect.infrastructure.docker-container - Pulling image '<<url>>/<<image>>:prefect2'...
15:24:32.990 | ERROR   | prefect.agent - Failed to submit flow run '6acdbe6c-56f7-49fa-bf61-792d0f1d3df9' to infrastructure.
Traceback (most recent call last):
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/docker/api/client.py", line 268, in _raise_for_status
    response.raise_for_status()
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: http+docker://localhost/v1.41/images/create?tag=prefect2&fromImage=<<url>>%2F<<image>>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/prefect/agent.py", line 206, in submit_run
    await self.task_group.start(submit_flow_run, flow_run, infrastructure)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 807, in start
    return await future
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/prefect/infrastructure/submission.py", line 48, in submit_flow_run
    return await infrastructure.run(task_status=task_status)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/prefect/infrastructure/docker.py", line 239, in run
    container_id = await run_sync_in_worker_thread(self._create_and_start_container)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/prefect/utilities/asyncutils.py", line 57, in run_sync_in_worker_thread
    return await anyio.to_thread.run_sync(call, cancellable=True)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/anyio/to_thread.py", line 31, in run_sync
    return await get_asynclib().run_sync_in_worker_thread(
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
    return await future
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 867, in run
    result = context.run(func, *args)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/prefect/infrastructure/docker.py", line 281, in _create_and_start_container
    self._pull_image(docker_client)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/prefect/infrastructure/docker.py", line 392, in _pull_image
    return docker_client.images.pull(image, tag)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/docker/models/images.py", line 465, in pull
    pull_log = self.client.api.pull(
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/docker/api/image.py", line 429, in pull
    self._raise_for_status(response)
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/docker/api/client.py", line 270, in _raise_for_status
    raise create_api_error_from_http_exception(e) from e
  File "/workspace/tmp/venv/prefect2/lib/python3.10/site-packages/docker/errors.py", line 39, in create_api_error_from_http_exception
    raise cls(e, response=response, explanation=explanation) from e
docker.errors.APIError: 500 Server Error for http+docker://localhost/v1.41/images/create?tag=prefect2&fromImage=<<url>>%2F<<image>>: Internal Server Error ("Head "https://<<url>>/v2/<<image>>/manifests/prefect2": no basic auth credentials")
```

Using the same credentials with `docker login...` works totally fine. We are hosting our registry on-prem with Nexus 3, which should make no difference dockerhub example from @anna-geller, because we are specifing the url inside the Docker Registry block.
When inspecting the Docker Container block the registry is visible:
![image](https://user-images.githubusercontent.com/18382402/187693211-8ea2d2e9-df2a-4020-b565-de65aab5d24c.png)

Version
```
Version:             2.3.0
API version:         0.8.0
Python version:      3.10.4
Git commit:          8d9316c0
Built:               Tue, Aug 30, 2022 5:30 PM
OS/Arch:             linux/x86_64
Profile:             default
Server type:         ephemeral
Server:
  Database:          sqlite
  SQLite version:    3.34.1
```

I'm also experiencing this. In fact, I get the 404 even if the docker image is already available locally. Very frustrating
How to about Prefect 2.4.x?
> How to about Prefect 2.4.x?

The same.

It happens because `DockerContainer` and `DockerRegistry` have different `docker_client`. 
And when `DockerContainer` pulls image it calls `DockerContainer.image_registry.login()`, so authentication passes successfully inside `DockerContainer.image_registry` but then `DockerContainer` still uses its own docker-client which is still without authentication.

Resolving patch:
diff --git a/src/prefect/infrastructure/docker.py b/src/prefect/infrastructure/docker.py
--- a/src/prefect/infrastructure/docker.py
+++ b/src/prefect/infrastructure/docker.py
@@ -46,15 +46,22 @@ class BaseDockerLogin(Block, ABC):
     _block_schema_capabilities = ["docker-login"]
 
     @abstractmethod
-    async def login() -> None:
+    async def login(self) -> "DockerClient":
         """
-        Log in with `docker login`, persisting credentials.
+        Log in and return an authenticated `DockerClient`.
+        (DEPRECATED) Use `get_docker_client` instead of `login`.
         """
 
-    def _login(self, username, password, registry_url, reauth):
+    @abstractmethod
+    async def get_docker_client(self) -> "DockerClient":
+        """
+        Log in and return an authenticated `DockerClient`.
+        """
+
+    def _login(self, username, password, registry_url, reauth) -> "DockerClient":
         client = self._get_docker_client()
 
-        return client.login(
+        client.login(
             username=username,
             password=password,
             registry=registry_url,
@@ -63,7 +70,10 @@ def _login(self, username, password, registry_url, reauth):
             reauth=reauth,
         )
 
-    def _get_docker_client(self):
+        return client
+
+    @staticmethod
+    def _get_docker_client():
         try:
 
             with warnings.catch_warnings():
@@ -87,8 +97,7 @@ class DockerRegistry(BaseDockerLogin):
     """
     Connects to a Docker registry.
 
-    Requires a Docker Engine to be connectable. Login information is persisted to disk
-    at the Docker default location.
+    Requires a Docker Engine to be connectable.
 
     Attributes:
         username: The username to log into the registry with.
@@ -117,8 +126,17 @@ class DockerRegistry(BaseDockerLogin):
     )
 
     @sync_compatible
-    async def login(self):
-        return await run_sync_in_worker_thread(
+    async def login(self) -> "DockerClient":
+        warnings.warn(
+            "`login` is deprecated. Instead, use `get_docker_client` to obtain an authenticated `DockerClient`.",
+            category=DeprecationWarning,
+            stacklevel=3,
+        )
+        return await self.get_docker_client()
+
+    @sync_compatible
+    async def get_docker_client(self) -> "DockerClient":
+        client = await run_sync_in_worker_thread(
             self._login,
             self.username,
             self.password.get_secret_value(),
@@ -126,6 +144,8 @@ async def login(self):
             self.reauth,
         )
 
+        return client
+
 
 class DockerContainerResult(InfrastructureResult):
     """Contains information about a completed Docker container"""
@@ -148,6 +168,8 @@ class DockerContainer(Infrastructure):
             Defaults to the Prefect image.
         image_pull_policy: Specifies if the image should be pulled. One of 'ALWAYS',
             'NEVER', 'IF_NOT_PRESENT'.
+        image_registry: A `DockerRegistry` block containing credentials to use if `image` is stored in a private
+            image registry.
         labels: An optional dictionary of labels, mapping name to value.
         name: An optional name for the container.
         network_mode: Set the network mode for the created container. Defaults to 'host'
@@ -285,7 +307,13 @@ def _build_container_settings(
         )
 
     def _create_and_start_container(self) -> "Container":
-        docker_client = self._get_client()
+        if self.image_registry:
+            # If an image registry block was supplied, load an authenticated Docker
+            # client from the block. Otherwise, use an unauthenticated client to
+            # pull images from public registries.
+            docker_client = self.image_registry.get_docker_client()
+        else:
+            docker_client = self._get_client()
 
         container_settings = self._build_container_settings(docker_client)
 
@@ -400,8 +428,7 @@ def _pull_image(self, docker_client: "DockerClient"):
         Pull the image we're going to use to create the container.
         """
         image, tag = self._get_image_and_tag()
-        if self.image_registry:
-            self.image_registry.login()
+
         return docker_client.images.pull(image, tag)
 
     def _create_container(self, docker_client: "DockerClient", **kwargs) -> "Container":


Test patch:
diff --git a/tests/blocks/checksums.json b/tests/blocks/checksums.json
--- a/tests/blocks/checksums.json
+++ b/tests/blocks/checksums.json
@@ -13,8 +13,8 @@
     "smb": "sha256:d5746a564cf05bec13770a882f76ca8e8f562c8bdc7a6d2d45732b079bf72045",
     "github": "sha256:4bf7c70ac6ff317928ababd19becb408dd4df53d7594e4a0f6756d866102f7ac",
     "kubernetes-cluster-config": "sha256:90d421e948bfbe4cdc98b124995f0edd0f84b0837549ad1390423bad8e31cf3b",
-    "docker-registry": "sha256:356410398f9957e9c96af04ae28875b71ca6174ac0978950b5f3641b21dc8544",
-    "docker-container": "sha256:77cf9424bd42cd96851b7363cc21279546a73e55a61cd815c54e279e57206a2a",
+    "docker-registry": "sha256:6db1457676eee0b54ca2834b06f80a80f7c90112e64f1bdd26afb2e62fcceed9",
+    "docker-container": "sha256:ee3348b26a5d0e0226e2d3cf6f2e85a3592c3b19fb8a988dfccceee972b9b9eb",
     "kubernetes-job": "sha256:686f931093d8fa3a80dee6eb66516be7b022bf29c44a38766da8571f25fede8b",
     "process": "sha256:227b03037cf895d9a70603554b792f573e7e59b81a585d51a43ec1f826e81da3"
 }
diff --git a/tests/infrastructure/test_docker_container.py b/tests/infrastructure/test_docker_container.py
--- a/tests/infrastructure/test_docker_container.py
+++ b/tests/infrastructure/test_docker_container.py
@@ -223,10 +223,7 @@ def test_allows_unsetting_environment_variables(
     assert "PREFECT_TEST_MODE" not in call_env
 
 
-def test_uses_image_registry_setting(
-    mock_docker_client,
-):
-
+def test_uses_image_registry_setting(mock_docker_client):
     DockerContainer(
         command=["echo", "hello"],
         image_registry=DockerRegistry(
@@ -235,11 +232,32 @@ def test_uses_image_registry_setting(
         image_pull_policy="ALWAYS",
     ).run()
 
+    # ensure that login occurs when DockerContainer is provided an
+    # image registry
     mock_docker_client.login.assert_called_once_with(
         username="foo", password="bar", registry="example.test", reauth=True
     )
 
 
+def test_uses_image_registry_client(mock_docker_client, monkeypatch):
+    container = DockerContainer(
+        command=["echo", "hello"],
+        image_registry=DockerRegistry(
+            username="foo", password="bar", registry_url="example.test"
+        ),
+        image_pull_policy="ALWAYS",
+    )
+
+    # ensure that DockerContainer is asking for an authenticated
+    # DockerClient from DockerRegistry.
+    mock_get_client = MagicMock()
+    mock_get_client.return_value = mock_docker_client
+    monkeypatch.setattr(container.image_registry, "get_docker_client", mock_get_client)
+    container.run()
+    mock_get_client.assert_called_once()
+    mock_docker_client.images.pull.assert_called_once()
+
+
 async def test_uses_image_registry_setting_after_save(
     mock_docker_client,
 ):
@@ -257,7 +275,6 @@ async def test_uses_image_registry_setting_after_save(
     ).save("test-docker-container")
 
     container = await DockerContainer.load("test-docker-container")
-
     await container.run()
 
     mock_docker_client.login.assert_called_once_with(
@@ -265,6 +282,29 @@ async def test_uses_image_registry_setting_after_save(
     )
 
 
+async def test_uses_image_registry_client_after_save(mock_docker_client, monkeypatch):
+    await DockerRegistry(
+        username="foo", password="bar", registry_url="example.test"
+    ).save("test-docker-registry")
+
+    await DockerContainer(
+        command=["echo", "hello"],
+        image_registry=await DockerRegistry.load("test-docker-registry"),
+        image_pull_policy="ALWAYS",
+    ).save("test-docker-container")
+
+    container = await DockerContainer.load("test-docker-container")
+
+    # ensure that the saved and reloaded DockerContainer is asking for an authenticated
+    # DockerClient from DockerRegistry.
+    mock_get_client = MagicMock()
+    mock_get_client.return_value = mock_docker_client
+    monkeypatch.setattr(container.image_registry, "get_docker_client", mock_get_client)
+    await container.run()
+    mock_get_client.assert_called_once()
+    mock_docker_client.images.pull.assert_called_once()
+
+
 @pytest.mark.parametrize("localhost", ["localhost", "127.0.0.1"])
 def test_network_mode_defaults_to_host_if_using_localhost_api_on_linux(
     mock_docker_client, localhost, monkeypatch