Repository: fractal-analytics-platform/fractal-client (language: python)

Issue:
Implement IMPORT/EXPORT for workflow commands
at the moment we raise "NotImplementedError".


Discussion / hints:

Fix patch:
diff --git a/fractal_client/cmd/__init__.py b/fractal_client/cmd/__init__.py
index ec633c3ef..6d0daf1de 100644
--- a/fractal_client/cmd/__init__.py
+++ b/fractal_client/cmd/__init__.py
@@ -36,6 +36,8 @@
 from ._workflow import patch_workflowtask
 from ._workflow import post_workflow
 from ._workflow import post_workflowtask
+from ._workflow import workflow_export
+from ._workflow import workflow_import
 from fractal_client import __VERSION__
 
 
@@ -234,6 +236,14 @@ def workflow(
         parameters = ["project_id", "workflow_id", "workflow_task_id"]
         function_kwargs = get_kwargs(parameters, kwargs)
         iface = delete_workflowtask(client, **function_kwargs)
+    elif subcmd == "import":
+        parameters = ["project_id", "json_file"]
+        function_kwargs = get_kwargs(parameters, kwargs)
+        iface = workflow_import(client, batch=batch, **function_kwargs)
+    elif subcmd == "export":
+        parameters = ["project_id", "workflow_id", "json_file"]
+        function_kwargs = get_kwargs(parameters, kwargs)
+        iface = workflow_export(client, **function_kwargs)
     else:
         raise NoCommandError(f"Command workflow {subcmd} not found")
     return iface
diff --git a/fractal_client/cmd/_workflow.py b/fractal_client/cmd/_workflow.py
index f03d1cd11..15987a71a 100644
--- a/fractal_client/cmd/_workflow.py
+++ b/fractal_client/cmd/_workflow.py
@@ -222,9 +222,76 @@ def patch_workflow(
     return Interface(retcode=0, data=new_workflow)
 
 
-def workflow_import() -> None:
-    raise NotImplementedError
+def workflow_import(
+    client: AuthClient, *, project_id: int, json_file: str, batch: bool = False
+) -> Interface:
+    with Path(json_file).open("r") as f:
+        workflow = json.load(f)
+
+    res = client.post(
+        f"{settings.BASE_URL}/project/{project_id}/workflow/import/",
+        json=workflow,
+    )
+    wf_read = check_response(res, expected_status_code=201)
+
+    warnings = [
+        workflow_task["task"]["source"]
+        for workflow_task in wf_read["task_list"]
+        if workflow_task["task"]["owner"]
+    ]
+    if warnings:
+        sources_str = ", ".join([f"'{s}'" for s in warnings])
+        logging.warning(
+            "This workflow includes custom tasks (the ones with sources: "
+            f"{sources_str}), which are not meant to be portable; "
+            "importing this workflow may not work as expected."
+        )
+
+    from devtools import debug
+
+    debug(batch)
+
+    if batch:
+        datastr = f"{wf_read['id']}"
+        for wftask in wf_read["task_list"]:
+            datastr += f" {wftask['id']}"
+        return Interface(retcode=0, data=datastr)
+    else:
+        return Interface(retcode=0, data=wf_read)
+
 
+def workflow_export(
+    client: AuthClient,
+    *,
+    project_id: int,
+    workflow_id: int,
+    json_file: str,
+) -> Interface:
+    res = client.get(
+        (
+            f"{settings.BASE_URL}/project/{project_id}/"
+            f"workflow/{workflow_id}/export/"
+        ),
+    )
+    workflow = check_response(res, expected_status_code=200)
 
-def workflow_export() -> None:
-    raise NotImplementedError
+    warnings = [
+        workflow_task["task"]["source"]
+        for workflow_task in workflow["task_list"]
+        if not workflow_task["task"]["source"].startswith(
+            ("pip_local:", "pip_remote:")
+        )
+    ]
+    if warnings:
+        sources_str = ", ".join([f"'{s}'" for s in warnings])
+        logging.warning(
+            "This workflow includes custom tasks (the ones with sources: "
+            f"{sources_str}), which are not meant to be portable; "
+            "re-importing this workflow may not work as expected."
+        )
+
+    with Path(json_file).open("w") as f:
+        json.dump(workflow, f, indent=2)
+    return Interface(
+        retcode=0, data=f"Workflow {workflow_id} exported at {json_file}"
+    )
diff --git a/poetry.lock b/poetry.lock
index 2424ab51b..df51972e7 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
 
 [[package]]
 name = "aiosqlite"
@@ -328,13 +328,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 
 [[package]]
 name = "cloudpickle"
-version = "2.2.1"
-description = "Extended pickling support for Python objects"
+version = "3.0.0"
+description = "Pickler class to extend the standard pickle.Pickler functionality"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.8"
 files = [
-    {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"},
-    {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"},
+    {file = "cloudpickle-3.0.0-py3-none-any.whl", hash = "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7"},
+    {file = "cloudpickle-3.0.0.tar.gz", hash = "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"},
 ]
 
 [[package]]
@@ -575,18 +575,18 @@ files = [
 
 [[package]]
 name = "fastapi"
-version = "0.109.2"
+version = "0.110.1"
 description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "fastapi-0.109.2-py3-none-any.whl", hash = "sha256:2c9bab24667293b501cad8dd388c05240c850b58ec5876ee3283c47d6e1e3a4d"},
-    {file = "fastapi-0.109.2.tar.gz", hash = "sha256:f3817eac96fe4f65a2ebb4baa000f394e55f5fccdaf7f75250804bc58f354f73"},
+    {file = "fastapi-0.110.1-py3-none-any.whl", hash = "sha256:5df913203c482f820d31f48e635e022f8cbfe7350e4830ef05a3163925b1addc"},
+    {file = "fastapi-0.110.1.tar.gz", hash = "sha256:6feac43ec359dfe4f45b2c18ec8c94edb8dc2dfc461d417d9e626590c071baad"},
 ]
 
 [package.dependencies]
 pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0"
-starlette = ">=0.36.3,<0.37.0"
+starlette = ">=0.37.2,<0.38.0"
 typing-extensions = ">=4.8.0"
 
 [package.extras]
@@ -620,13 +620,13 @@ sqlalchemy = ["fastapi-users-db-sqlalchemy (>=6.0.0)"]
 
 [[package]]
 name = "filelock"
-version = "3.13.3"
+version = "3.13.4"
 description = "A platform independent file lock."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "filelock-3.13.3-py3-none-any.whl", hash = "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb"},
-    {file = "filelock-3.13.3.tar.gz", hash = "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"},
+    {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"},
+    {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"},
 ]
 
 [package.extras]
@@ -645,18 +645,18 @@ develop = false
 
 [package.dependencies]
 aiosqlite = "^0.19.0"
-alembic = "^1.9.1"
+alembic = "^1.13.1"
 bcrypt = "4.0.1"
-cloudpickle = ">=2.2.1,<2.3.0"
+cloudpickle = ">=3.0.0,<3.1.0"
 clusterfutures = "^0.5"
-fastapi = "^0.109.0"
+fastapi = "^0.110.0"
 fastapi-users = {version = "^12.1.0", extras = ["oauth"]}
 packaging = "^23.2"
 pydantic = ">=1.10.8,<2"
-python-dotenv = "^0.21.0"
+python-dotenv = "^1.0.0"
 sqlalchemy = {version = ">=2.0.23,<2.1", extras = ["asyncio"]}
-sqlmodel = "^0.0.14"
-uvicorn = "^0.27.0"
+sqlmodel = "^0.0.16"
+uvicorn = "^0.29.0"
 
 [package.extras]
 gunicorn = ["gunicorn (>=21.2.0,<22.0.0)"]
@@ -666,7 +666,7 @@ postgres = ["asyncpg (>=0.29.0,<0.30.0)", "psycopg2 (>=2.9.5,<3.0.0)"]
 type = "git"
 url = "https://github.com/fractal-analytics-platform/fractal-server.git"
 reference = "main"
-resolved_reference = "b7db36079cb2ddd1ecb70b3e607b33a193df3c06"
+resolved_reference = "84d3db7314e09f9cc6fcb6124e5da050d18564fc"
 
 [[package]]
 name = "ghp-import"
@@ -1497,13 +1497,13 @@ six = ">=1.5"
 
 [[package]]
 name = "python-dotenv"
-version = "0.21.1"
+version = "1.0.1"
 description = "Read key-value pairs from a .env file and set them as environment variables"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "python-dotenv-0.21.1.tar.gz", hash = "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49"},
-    {file = "python_dotenv-0.21.1-py3-none-any.whl", hash = "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"},
+    {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"},
+    {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
 ]
 
 [package.extras]
@@ -1864,13 +1864,13 @@ sqlcipher = ["sqlcipher3_binary"]
 
 [[package]]
 name = "sqlmodel"
-version = "0.0.14"
+version = "0.0.16"
 description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness."
 optional = false
 python-versions = ">=3.7,<4.0"
 files = [
-    {file = "sqlmodel-0.0.14-py3-none-any.whl", hash = "sha256:accea3ff5d878e41ac439b11e78613ed61ce300cfcb860e87a2d73d4884cbee4"},
-    {file = "sqlmodel-0.0.14.tar.gz", hash = "sha256:0bff8fc94af86b44925aa813f56cf6aabdd7f156b73259f2f60692c6a64ac90e"},
+    {file = "sqlmodel-0.0.16-py3-none-any.whl", hash = "sha256:b972f5d319580d6c37ecc417881f6ec4d1ad3ed3583d0ac0ed43234a28bf605a"},
+    {file = "sqlmodel-0.0.16.tar.gz", hash = "sha256:966656f18a8e9a2d159eb215b07fb0cf5222acfae3362707ca611848a8a06bd1"},
 ]
 
 [package.dependencies]
@@ -1879,13 +1879,13 @@ SQLAlchemy = ">=2.0.0,<2.1.0"
 
 [[package]]
 name = "starlette"
-version = "0.36.3"
+version = "0.37.2"
 description = "The little ASGI library that shines."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "starlette-0.36.3-py3-none-any.whl", hash = "sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044"},
-    {file = "starlette-0.36.3.tar.gz", hash = "sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080"},
+    {file = "starlette-0.37.2-py3-none-any.whl", hash = "sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee"},
+    {file = "starlette-0.37.2.tar.gz", hash = "sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823"},
 ]
 
 [package.dependencies]
@@ -1947,13 +1947,13 @@ zstd = ["zstandard (>=0.18.0)"]
 
 [[package]]
 name = "uvicorn"
-version = "0.27.1"
+version = "0.29.0"
 description = "The lightning-fast ASGI server."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "uvicorn-0.27.1-py3-none-any.whl", hash = "sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4"},
-    {file = "uvicorn-0.27.1.tar.gz", hash = "sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a"},
+    {file = "uvicorn-0.29.0-py3-none-any.whl", hash = "sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de"},
+    {file = "uvicorn-0.29.0.tar.gz", hash = "sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0"},
 ]
 
 [package.dependencies]
@@ -2043,4 +2043,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "564f755159caa6386e2794b40e5cef65bdd6f4cddf51c489d5f092994f80a09f"
+content-hash = "54c9e44c92835523df59f8b75a24652b9b2f96a7009cd23aa27a1916867d5afc"
diff --git a/pyproject.toml b/pyproject.toml
index bc2984cff..4b5c61aa5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,7 +24,7 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-python-dotenv = "^0.21.0"
+python-dotenv = "^1.0.0"
 httpx = "^0.23.0"
 PyJWT = "^2.4.0"
 packaging = "^23.1"


Test patch:
diff --git a/tests/data/import-export/workflow.json b/tests/data/import-export/workflow.json
index 225ec7df2..d8ec31b90 100644
--- a/tests/data/import-export/workflow.json
+++ b/tests/data/import-export/workflow.json
@@ -1,27 +1,21 @@
 {
-    "name": "MyWorkflow",
+    "name": "MyWorkflow-V2",
     "task_list": [
         {
-            "args": {
-                "message": "task 0",
-                "arg": "value"
+            "args_parallel": {
+                "message_x": "task_x",
+                "arg_x": "value_x"
             },
-            "meta": {"key": "value"},
-            "task": {
-                "source": "pip_local:fractal_tasks_dummy:0.1.0:::dummy"
-            }
-        },
-        {
-            "args": {
-                "message": "task 1"
+            "args_non_parallel": {
+                "message_y": "task_y",
+                "arg_y": "value_y"
             },
+            "meta_parallel": {"key1": "value1"},
+            "meta_non_parallel": {"key2": "value2"},
+            "order": 0,
+            "is_legacy_task": false,
             "task": {
-                "source": "pip_local:fractal_tasks_dummy:0.1.0:::dummy_parallel"
-            }
-        },
-        {
-            "task": {
-                "source": "custom_source"
+                "source": "PKG_SOURCE:dummy2"
             }
         }
     ]
diff --git a/tests/data/import-export/workflow_2.json b/tests/data/import-export/workflow_2.json
index 1d7499957..8f51f079a 100644
--- a/tests/data/import-export/workflow_2.json
+++ b/tests/data/import-export/workflow_2.json
@@ -1,27 +1,21 @@
 {
-    "name": "MyWorkflow2",
+    "name": "MyWorkflow-V2-2",
     "task_list": [
         {
-            "args": {
-                "message": "task 0",
-                "arg": "value"
+            "args_parallel": {
+                "message_x": "task_x",
+                "arg_x": "value_x"
             },
-            "meta": {"key": "value"},
-            "task": {
-                "source": "pip_local:fractal_tasks_dummy:0.1.0:::dummy"
-            }
-        },
-        {
-            "args": {
-                "message": "task 1"
+            "args_non_parallel": {
+                "message_y": "task_y",
+                "arg_y": "value_y"
             },
+            "meta_parallel": {"key1": "value1"},
+            "meta_non_parallel": {"key2": "value2"},
+            "order": 0,
+            "is_legacy_task": false,
             "task": {
-                "source": "pip_local:fractal_tasks_dummy:0.1.0:::dummy_parallel"
-            }
-        },
-        {
-            "task": {
-                "source": "custom_source"
+                "source": "PKG_SOURCE:dummy2"
             }
         }
     ]
diff --git a/tests/test_workflow.py b/tests/test_workflow.py
index 966e3906c..4c062c688 100644
--- a/tests/test_workflow.py
+++ b/tests/test_workflow.py
@@ -530,3 +530,99 @@ def test_workflow_edit_task(
     workflow_task = res.data
     assert workflow_task["meta_non_parallel"] == META_NON_PARALLEL
     assert workflow_task["args_non_parallel"] == ARGS_NON_PARALLEL
+
+
+def test_workflow_import(
+    register_user,
+    invoke,
+    testdata_path: Path,
+    task_factory,
+    caplog,
+):
+
+    # create project
+    PROJECT_NAME = "project_name"
+    res_pj = invoke(f"project new {PROJECT_NAME}")
+    assert res_pj.retcode == 0
+    project_id = res_pj.data["id"]
+
+    task_factory(name="task", source="PKG_SOURCE:dummy2", owner="exact-lab")
+
+    # import workflow into project
+    filename = str(testdata_path / "import-export/workflow.json")
+    with open(filename, "r") as f:
+        debug(f.read())
+    res = invoke(
+        f"workflow import --project-id {project_id} --json-file {filename}"
+    )
+    debug(res.data)
+    assert res.retcode == 0
+    assert caplog.records[-1].msg == (
+        "This workflow includes custom tasks (the ones with sources: "
+        "'PKG_SOURCE:dummy2'), which are not meant to be portable; "
+        "importing this workflow may not work as expected."
+    )
+
+    imported_workflow = res.data
+
+    # get the workflow from the server, and check that it is the same
+    workflow_id = res.data["id"]
+    res = invoke(f"workflow show {project_id} {workflow_id}")
+    assert res.retcode == 0
+    assert res.data == imported_workflow
+
+    # import workflow into project, with --batch
+    filename = str(testdata_path / "import-export/workflow_2.json")
+    res = invoke(
+        f"--batch workflow import --project-id {project_id} "
+        f"--json-file {filename}"
+    )
+    assert res.retcode == 0
+    assert res.data == "2 2"
+    assert caplog.records[-1].msg == (
+        "This workflow includes custom tasks (the ones with sources: "
+        "'PKG_SOURCE:dummy2'), which are not meant to be portable; "
+        "importing this workflow may not work as expected."
+    )
+
+
+def test_workflow_export(
+    register_user,
+    invoke,
+    workflow_factory,
+    tmp_path: Path,
+    task_factory,
+    caplog,
+):
+
+    res = invoke("project new testproject")
+    assert res.retcode == 0
+    project_id = res.data["id"]
+
+    NAME = "WorkFlow"
+    wf = workflow_factory(project_id=project_id, name=NAME)
+    prj_id = wf.project_id
+    wf_id = wf.id
+    filename = str(tmp_path / "exported_wf.json")
+
+    task = task_factory(owner="exact-lab")
+    res = invoke(f"workflow add-task {prj_id} {wf_id} --task-id {task.id}")
+    assert res.retcode == 0
+
+    res = invoke(f"workflow export {prj_id} {wf_id} --json-file {filename}")
+    assert res.retcode == 0
+    assert caplog.records[-1].msg == (
+        "This workflow includes custom tasks (the ones with sources: "
+        f"'{task.source}'), which are not meant to be portable; "
+        "re-importing this workflow may not work as expected."
+    )
+    debug(res.data)
+    with open(filename, "r") as f:
+        exported_wf = json.load(f)
+        assert exported_wf["name"] == NAME
+        assert "id" not in exported_wf
+        assert "project_id" not in exported_wf
+        for wftask in exported_wf["task_list"]:
+            assert "id" not in wftask
+            assert "task_id" not in wftask
+            assert "workflow_id" not in wftask