Repository: PrefectHQ/prefect

Issue:
Make docker server fungible
## Use Case
I'd like to be able to keep data saved in the docker postgres database across docker restarts.

At the moment, a persistent docker volume is not created on the host and any data like ETL registrations and previous runs is lost when docker restarts.

## Solution
Either always create a docker volume in the docker-compose.yml, or add an option in the cli to create a docker volume.

## Alternatives
I've considered maybe a snapshot mechanism from within the prefect service but adding the docker volume seems like the easiest approach.



Discussion:

Resolving patch:
diff --git a/src/prefect/cli/server.py b/src/prefect/cli/server.py
--- a/src/prefect/cli/server.py
+++ b/src/prefect/cli/server.py
@@ -48,6 +48,7 @@ def make_env(fname=None):
         POSTGRES_USER=config.server.database.username,
         POSTGRES_PASSWORD=config.server.database.password,
         POSTGRES_DB=config.server.database.name,
+        POSTGRES_DATA_PATH=config.server.database.volume_path,
     )
 
     UI_ENV = dict(GRAPHQL_URL=config.server.ui.graphql_url)
@@ -178,6 +179,19 @@ def server():
     is_flag=True,
     hidden=True,
 )
+@click.option(
+    "--use-volume",
+    help="Enable the use of a volume for the postgres service",
+    is_flag=True,
+    hidden=True,
+)
+@click.option(
+    "--volume-path",
+    help="A path to use for the postgres volume",
+    default=config.server.database.volume_path,
+    type=str,
+    hidden=True,
+)
 def start(
     version,
     skip_pull,
@@ -193,31 +207,37 @@ def start(
     no_graphql_port,
     no_ui_port,
     no_server_port,
+    use_volume,
+    volume_path,
 ):
     """
     This command spins up all infrastructure and services for the Prefect Core server
 
     \b
     Options:
-        --version, -v   TEXT    The server image versions to use (for example, '0.10.0' or 'master')
-                                Defaults to the current installed Prefect version.
-        --skip-pull             Flag to skip pulling new images (if available)
-        --no-upgrade, -n        Flag to avoid running a database upgrade when the database spins up
-        --no-ui, -u             Flag to avoid starting the UI
+        --version, -v       TEXT    The server image versions to use (for example, '0.10.0' or 'master')
+                                    Defaults to the current installed Prefect version.
+        --skip-pull                 Flag to skip pulling new images (if available)
+        --no-upgrade, -n            Flag to avoid running a database upgrade when the database spins up
+        --no-ui, -u                 Flag to avoid starting the UI
 
     \b
-        --postgres-port TEXT    Port used to serve Postgres, defaults to '5432'
-        --hasura-port   TEXT    Port used to serve Hasura, defaults to '3001'
-        --graphql-port  TEXT    Port used to serve the GraphQL API, defaults to '4001'
-        --ui-port       TEXT    Port used to serve the UI, defaults to '8080'
-        --server-port   TEXT    Port used to serve the Core server, defaults to '4200'
+        --postgres-port     TEXT    Port used to serve Postgres, defaults to '5432'
+        --hasura-port       TEXT    Port used to serve Hasura, defaults to '3001'
+        --graphql-port      TEXT    Port used to serve the GraphQL API, defaults to '4001'
+        --ui-port           TEXT    Port used to serve the UI, defaults to '8080'
+        --server-port       TEXT    Port used to serve the Core server, defaults to '4200'
 
     \b
-        --no-postgres-port      Disable port map of Postgres to host
-        --no-hasura-port        Disable port map of Hasura to host
-        --no-graphql-port       Disable port map of the GraphQL API to host
-        --no-ui-port            Disable port map of the UI to host
-        --no-server-port        Disable port map of the Core server to host
+        --no-postgres-port          Disable port map of Postgres to host
+        --no-hasura-port            Disable port map of Hasura to host
+        --no-graphql-port           Disable port map of the GraphQL API to host
+        --no-ui-port                Disable port map of the UI to host
+        --no-server-port            Disable port map of the Core server to host
+
+    \b
+        --use-volume                Enable the use of a volume for the Postgres service
+        --volume-path       TEXT    A path to use for the Postgres volume, defaults to '~/.prefect/pg_data'
     """
 
     docker_dir = Path(__file__).parents[0]
@@ -231,6 +251,7 @@ def start(
         or no_ui_port
         or no_server_port
         or platform_is_linux()
+        or not use_volume
     ):
         temp_dir = tempfile.gettempdir()
         temp_path = os.path.join(temp_dir, "docker-compose.yml")
@@ -261,6 +282,9 @@ def start(
                         "host.docker.internal:{}".format(docker_internal_ip)
                     ]
 
+            if not use_volume:
+                del y["services"]["postgres"]["volumes"]
+
         with open(temp_path, "w") as f:
             y = yaml.safe_dump(y, f)
 
@@ -274,6 +298,7 @@ def start(
             "server.graphql.host_port": str(graphql_port),
             "server.ui.host_port": str(ui_port),
             "server.host_port": str(server_port),
+            "server.database.volume_path": volume_path,
         }
     ):
         env = make_env()


Test patch:
diff --git a/tests/cli/test_server.py b/tests/cli/test_server.py
--- a/tests/cli/test_server.py
+++ b/tests/cli/test_server.py
@@ -42,6 +42,7 @@ def test_make_env_config_vars():
             "server.database.password": "password",
             "server.database.name": "db",
             "server.hasura.host_port": "7",
+            "server.database.volume_path": "test/path",
         }
     ):
         env = make_env()
@@ -59,6 +60,7 @@ def test_make_env_config_vars():
         assert env["POSTGRES_PASSWORD"] == "password"
         assert env["POSTGRES_DB"] == "db"
         assert env["HASURA_HOST_PORT"] == "7"
+        assert env["POSTGRES_DATA_PATH"] == "test/path"
 
 
 def test_server_start(monkeypatch, macos_platform):
@@ -263,3 +265,27 @@ def test_server_start_linux_host(monkeypatch, linux_platform):
         assert call_arg["services"][svc]["extra_hosts"] == [
             "host.docker.internal:172.17.0.1"
         ]
+
+
+def test_server_start_volume_options(monkeypatch, macos_platform):
+    check_call = MagicMock()
+    popen = MagicMock(side_effect=KeyboardInterrupt())
+    check_output = MagicMock()
+    monkeypatch.setattr("subprocess.Popen", popen)
+    monkeypatch.setattr("subprocess.check_call", check_call)
+    monkeypatch.setattr("subprocess.check_output", check_output)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        server, ["start", "--use-volume", "--volume-path", "test/path"],
+    )
+    assert result.exit_code == 1
+
+    assert check_call.called
+    assert popen.called
+    assert check_output.called
+
+    assert popen.call_args[0][0] == ["docker-compose", "up"]
+    assert popen.call_args[1].get("cwd")
+    assert popen.call_args[1].get("env")
+    assert popen.call_args[1]["env"].get("POSTGRES_DATA_PATH") == "test/path"