Repository: fractal-analytics-platform/fractal-client (language: python)

Issue:
use git submodule for common



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c4dd6415c..e3a2e8425 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,6 +20,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          submodules: "recursive"
           fetch-depth: 1
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -27,10 +28,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      #- name: Run pre-commit
-      #  if: ${{ matrix.python-version == '3.10' }}
-      #  uses: pre-commit/action@v3.0.0
-
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
         with:
@@ -64,6 +61,9 @@ jobs:
     needs: tests
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          fetch-depth: 1
 
       - uses: actions/setup-python@v3
         with:
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..36aa2b065
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "fractal/common"]
+	path = fractal/common
+	url = git@github.com:fractal-analytics-platform/fractal-common.git
diff --git a/fractal/.gitignore b/fractal/.gitignore
deleted file mode 100644
index bc9ad73a1..000000000
--- a/fractal/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-Temporary_folder_for_regression_test/
-monitoring.db
-jobs/wf*
-runinfo
diff --git a/fractal/client/authclient.py b/fractal/authclient.py
similarity index 100%
rename from fractal/client/authclient.py
rename to fractal/authclient.py
diff --git a/fractal/client/client.py b/fractal/client.py
similarity index 100%
rename from fractal/client/client.py
rename to fractal/client.py
diff --git a/fractal/client/__init__.py b/fractal/client/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/fractal/client/schemas.py b/fractal/client/schemas.py
deleted file mode 100644
index a71f1c127..000000000
--- a/fractal/client/schemas.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from ..common.schemas import *  # noqa: F401, F403
diff --git a/fractal/client/cmd/__init__.py b/fractal/cmd/__init__.py
similarity index 100%
rename from fractal/client/cmd/__init__.py
rename to fractal/cmd/__init__.py
diff --git a/fractal/client/cmd/_dataset.py b/fractal/cmd/_dataset.py
similarity index 96%
rename from fractal/client/cmd/_dataset.py
rename to fractal/cmd/_dataset.py
index 622e30368..1f5c1c0bd 100644
--- a/fractal/client/cmd/_dataset.py
+++ b/fractal/cmd/_dataset.py
@@ -7,16 +7,16 @@
 from rich.table import Table
 
 from ..authclient import AuthClient
+from ..common.schemas import DatasetRead
+from ..common.schemas import DatasetUpdate
+from ..common.schemas import ResourceCreate
+from ..common.schemas import ResourceRead
 from ..config import settings
 from ..interface import BaseInterface
 from ..interface import PrintInterface
 from ..interface import RichConsoleInterface
 from ..interface import RichJsonInterface
 from ..response import check_response
-from ..schemas import DatasetRead
-from ..schemas import DatasetUpdate
-from ..schemas import ResourceCreate
-from ..schemas import ResourceRead
 
 
 async def dataset_add_resource(
diff --git a/fractal/client/cmd/_project.py b/fractal/cmd/_project.py
similarity index 95%
rename from fractal/client/cmd/_project.py
rename to fractal/cmd/_project.py
index a38425999..4cc0ed130 100644
--- a/fractal/client/cmd/_project.py
+++ b/fractal/cmd/_project.py
@@ -5,16 +5,16 @@
 from rich.table import Table
 
 from ..authclient import AuthClient
+from ..common.schemas import DatasetCreate
+from ..common.schemas import DatasetRead
+from ..common.schemas import ProjectCreate
+from ..common.schemas import ProjectRead
 from ..config import settings
 from ..interface import BaseInterface
 from ..interface import PrintInterface
 from ..interface import RichConsoleInterface
 from ..interface import RichJsonInterface
 from ..response import check_response
-from ..schemas import DatasetCreate
-from ..schemas import DatasetRead
-from ..schemas import ProjectCreate
-from ..schemas import ProjectRead
 
 
 async def project_create(
diff --git a/fractal/client/cmd/_task.py b/fractal/cmd/_task.py
similarity index 96%
rename from fractal/client/cmd/_task.py
rename to fractal/cmd/_task.py
index 4c7d5066d..06f33ae34 100644
--- a/fractal/client/cmd/_task.py
+++ b/fractal/cmd/_task.py
@@ -4,15 +4,15 @@
 from typing import Optional
 
 from ..authclient import AuthClient
+from ..common.schemas import StateRead
+from ..common.schemas import TaskCollectPip
+from ..common.schemas import TaskRead
+from ..common.schemas import TaskUpdate
 from ..config import settings
 from ..interface import BaseInterface
 from ..interface import PrintInterface
 from ..interface import RichJsonInterface
 from ..response import check_response
-from ..schemas import StateRead
-from ..schemas import TaskCollectPip
-from ..schemas import TaskRead
-from ..schemas import TaskUpdate
 
 
 async def get_cached_task_by_name(name: str, client: AuthClient) -> int:
diff --git a/fractal/client/cmd/_workflow.py b/fractal/cmd/_workflow.py
similarity index 93%
rename from fractal/client/cmd/_workflow.py
rename to fractal/cmd/_workflow.py
index 0d20a1d8a..4bd541c24 100644
--- a/fractal/client/cmd/_workflow.py
+++ b/fractal/cmd/_workflow.py
@@ -4,18 +4,18 @@
 from typing import Optional
 
 from ..authclient import AuthClient
+from ..common.schemas import ApplyWorkflowCreate
+from ..common.schemas import WorkflowCreate
+from ..common.schemas import WorkflowRead
+from ..common.schemas import WorkflowTaskCreate
+from ..common.schemas import WorkflowTaskRead
+from ..common.schemas import WorkflowTaskUpdate
+from ..common.schemas import WorkflowUpdate
 from ..config import settings
 from ..interface import BaseInterface
 from ..interface import PrintInterface
 from ..interface import RichJsonInterface
 from ..response import check_response
-from ..schemas import ApplyWorkflowCreate
-from ..schemas import WorkflowCreate
-from ..schemas import WorkflowRead
-from ..schemas import WorkflowTaskCreate
-from ..schemas import WorkflowTaskRead
-from ..schemas import WorkflowTaskUpdate
-from ..schemas import WorkflowUpdate
 
 
 async def workflow_new(
diff --git a/fractal/common b/fractal/common
new file mode 160000
index 000000000..45f6906aa
--- /dev/null
+++ b/fractal/common
@@ -0,0 +1,1 @@
+Subproject commit 45f6906aaf8c91e6b5f37641113f6a1008527566
diff --git a/fractal/common/schemas/__init__.py b/fractal/common/schemas/__init__.py
deleted file mode 100644
index 3112f480f..000000000
--- a/fractal/common/schemas/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from sqlmodel import SQLModel
-
-from .manifest import *  # noqa: F403
-from .project import *  # noqa: F403
-from .state import *  # noqa: F403
-from .task import *  # noqa: F403
-from .workflow import *  # noqa: F403
-
-
-__all__ = (
-    (
-        "ApplyWorkflowBase",
-        "ApplyWorkflowCreate",
-        "ApplyWorkflowRead",
-    )
-    + project.__all__  # noqa: F405
-    + task.__all__  # noqa: F405
-    + workflow.__all__  # noqa: F405
-    + manifest.__all__  # noqa: F405
-    + state.__all__  # noqa: F405
-)
-
-
-class ApplyWorkflowBase(SQLModel):
-    project_id: int
-    input_dataset_id: int
-    output_dataset_id: Optional[int]
-    workflow_id: Optional[int]
-    overwrite_input: bool = False
-    worker_init: Optional[str]
-
-
-class ApplyWorkflowCreate(ApplyWorkflowBase):
-    pass
-
-
-class ApplyWorkflowRead(ApplyWorkflowBase):
-    id: int
-    start_timestamp: datetime
diff --git a/fractal/common/schemas/manifest.py b/fractal/common/schemas/manifest.py
deleted file mode 100644
index 1641f8504..000000000
--- a/fractal/common/schemas/manifest.py
+++ /dev/null
@@ -1,94 +0,0 @@
-from pathlib import Path
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-from typing import TypeVar
-
-from pydantic import BaseModel
-from pydantic import validator
-
-
-__all__ = ("TaskManifestV1", "ManifestV1")
-
-
-class _TaskManifestBase(BaseModel):
-    """
-    Base class for TaskManifest
-
-    Represents a task within a manfest
-
-    Attributes
-    ----------
-    name : str
-        the task name
-    executable : Path
-        path to the executable relative to the package root
-
-        Note: by package root we mean "as it will be installed". If a package
-        `Pkg` installs in the folder `pkg` the executable `pkg/executable.py`,
-        this attribute must contain only `executable.py`.
-    input_type : str
-        the input type accepted by the task
-    output_type : str
-        the output type returned by the task
-    default_args : Dict[str, Any]
-        an arbitrary, JSON-serializable dictionary containing the default
-        parameters that will be passed to the task
-    meta : Dict[str, Any]
-        additional information about the package, such as hash of the
-        executable, specific runtime requirements (e.g., need_gpu=True), etc.
-    """
-
-    name: str
-    executable: Path
-    input_type: str
-    output_type: str
-    default_args: Dict[str, Any]
-    meta: Optional[Dict[str, Any]]
-
-
-TaskManifestType = TypeVar("TaskManifestType", bound=_TaskManifestBase)
-
-
-class _ManifestBase(BaseModel):
-    """
-    Manifest base class
-
-    Packages containing tasks are required to include a special file
-    `__FRACTAL_MANIFEST__.json` in order to be discovered and used by Fractal.
-
-    This model class and the model classes it depends on provide the base
-    schema to read, write and validate manifests.
-
-    Attributes
-    ----------
-    manifest_version : str
-        a version string that provides indication for compatibility between
-        manifests as the schema evolves. This is for instance used by Fractal
-        to determine which subclass of the present base class needs be used to
-        read and validate the input.
-    task_list : List[TaskManifestType]
-        the list of tasks, represented as specified by subclasses of the
-        _TaskManifestBase (a.k.a. TaskManifestType)
-    """
-
-    manifest_version: str
-    task_list: List[TaskManifestType]  # type: ignore
-
-
-class TaskManifestV1(_TaskManifestBase):
-    pass
-
-
-class ManifestV1(_ManifestBase):
-    """
-    Manifest schema version 1
-    """
-
-    task_list: List[TaskManifestV1]
-
-    @validator("manifest_version")
-    def manifest_version_1(cls, value):
-        if value != "1":
-            raise ValueError("Wrong manifest version")
diff --git a/fractal/common/schemas/project.py b/fractal/common/schemas/project.py
deleted file mode 100644
index 218682728..000000000
--- a/fractal/common/schemas/project.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from pathlib import Path
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-
-from pydantic import validator
-from sqlmodel import SQLModel
-
-
-__all__ = (
-    "ProjectCreate",
-    "ProjectRead",
-    "DatasetUpdate",
-    "DatasetCreate",
-    "DatasetRead",
-    "ResourceCreate",
-    "ResourceRead",
-    "ResourceUpdate",
-)
-
-
-class _ProjectBase(SQLModel):
-    name: str
-    project_dir: str
-    read_only: bool = False
-
-
-class ProjectCreate(_ProjectBase):
-    default_dataset_name: Optional[str] = "default"
-
-    @validator("default_dataset_name")
-    def not_null(cls, value):
-        if not value:
-            value = "default"
-        return value
-
-
-class ProjectRead(_ProjectBase):
-    id: int
-    dataset_list: List["DatasetRead"] = []
-
-
-# DATASET
-
-
-class _DatasetBase(SQLModel):
-    name: str
-    project_id: Optional[int]
-    type: Optional[str]
-    meta: Dict[str, Any] = {}
-    read_only: Optional[bool] = False
-
-
-class DatasetUpdate(_DatasetBase):
-    name: Optional[str]  # type:ignore
-    meta: Optional[Dict[str, Any]] = None  # type:ignore
-
-
-class DatasetCreate(_DatasetBase):
-    pass
-
-
-class DatasetRead(_DatasetBase):
-    id: int
-    resource_list: List["ResourceRead"]
-
-
-# RESOURCE
-
-
-class _ResourceBase(SQLModel):
-    path: str
-    glob_pattern: Optional[str] = ""
-
-    @property
-    def glob_path(self) -> Path:
-        return Path(self.path) / self.glob_pattern
-
-
-class ResourceCreate(_ResourceBase):
-    pass
-
-
-class ResourceUpdate(_ResourceBase):
-    pass
-
-
-class ResourceRead(_ResourceBase):
-    id: int
-    dataset_id: int
-
-
-ProjectRead.update_forward_refs()
-DatasetRead.update_forward_refs()
diff --git a/fractal/common/schemas/state.py b/fractal/common/schemas/state.py
deleted file mode 100644
index e4ec3cb30..000000000
--- a/fractal/common/schemas/state.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from datetime import datetime
-from typing import Any
-from typing import Dict
-from typing import Optional
-
-from sqlmodel import SQLModel
-
-__all__ = (
-    "_StateBase",
-    "StateRead",
-)
-
-
-class _StateBase(SQLModel):
-    id: Optional[int]
-    data: Dict[str, Any]
-    timestamp: datetime
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    def sanitised_dict(self):
-        d = self.dict()
-        d["timestamp"] = str(self.timestamp)
-        return d
-
-
-class StateRead(_StateBase):
-    pass
diff --git a/fractal/common/schemas/task.py b/fractal/common/schemas/task.py
deleted file mode 100644
index e38d0d26f..000000000
--- a/fractal/common/schemas/task.py
+++ /dev/null
@@ -1,124 +0,0 @@
-from pathlib import Path
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Literal
-from typing import Optional
-
-from pydantic import BaseModel
-from pydantic import validator
-from sqlmodel import Field  # type: ignore
-from sqlmodel import SQLModel
-
-
-__all__ = (
-    "TaskCreate",
-    "TaskUpdate",
-    "TaskRead",
-    "TaskCollectPip",
-    "TaskCollectResult",
-    "TaskCollectStatus",
-)
-
-
-class _TaskBase(SQLModel):
-    """
-    Task base class
-
-    A Task is the elemental unit of a workflow, and must be a self-standing
-    executable.
-
-    Attributes
-    ----------
-    name: str
-        a human readable name for the task
-    command: str
-        the command(s) that executes the task
-    source: str
-        path or url to task source. This is the information is used to match
-        tasks across fractal installations when a workflow is imported.
-    input_type, output_type: str
-        the type of data the task expects as input, output, respectively.
-    default_args: Dict[str, Any]
-        dictionary (saved as JSON) of the default parameters of the task
-    """
-
-    name: str
-    command: str
-    source: str
-    input_type: str
-    output_type: str
-    default_args: Dict[str, Any] = Field(default={})
-    meta: Optional[Dict[str, Any]] = Field(default={})
-
-    class Config:
-        arbitrary_types_allowed = True
-
-
-class TaskUpdate(_TaskBase):
-    name: Optional[str]  # type:ignore
-    input_type: Optional[str]  # type:ignore
-    output_type: Optional[str]  # type:ignore
-    command: Optional[str]  # type:ignore
-    source: Optional[str]  # type:ignore
-    default_args: Optional[Dict[str, Any]] = None  # type:ignore
-
-
-class TaskRead(_TaskBase):
-    id: int
-
-
-class TaskCreate(_TaskBase):
-    pass
-
-
-class _TaskCollectBase(BaseModel):
-    pass
-
-
-class TaskCollectPip(_TaskCollectBase):
-    package: str
-    version: Optional[str]
-    python_version: str = "3.8"
-    package_extras: Optional[str]
-
-    @validator("package")
-    def package_path_absolute(cls, value):
-        if "/" in value:
-            package_path = Path(value)
-            if not package_path.is_absolute():
-                raise ValueError(
-                    f"Package path must be absolute: {package_path}"
-                )
-            if not package_path.exists():
-                raise ValueError(
-                    f"Package file does not exist: {package_path}"
-                )
-        return value
-
-
-class TaskCollectResult(_TaskCollectBase):
-    """
-    Temporary result of the collection process
-
-    This schema represents the partial result returned when collection is
-    initiated but it is continuing in the background.
-
-    Attributes:
-        package (str):
-            the package name
-        venv_path (Path):
-            the installation path relative to `FRACTAL_ROOT`
-        info (str):
-            arbitrary information
-    """
-
-    package: str
-    venv_path: Path
-    info: Optional[str]
-
-
-class TaskCollectStatus(_TaskCollectBase):
-    status: Literal["OK", "pending", "fail"]
-    task_list: Optional[List[TaskRead]] = Field(default=[])
-    log: Optional[str]
diff --git a/fractal/common/schemas/workflow.py b/fractal/common/schemas/workflow.py
deleted file mode 100644
index 0127fef76..000000000
--- a/fractal/common/schemas/workflow.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-
-from pydantic import validator
-from sqlmodel import SQLModel
-
-from .task import TaskRead
-
-
-__all__ = (
-    "WorkflowCreate",
-    "WorkflowRead",
-    "WorkflowUpdate",
-    "WorkflowTaskCreate",
-    "WorkflowTaskRead",
-    "WorkflowTaskUpdate",
-)
-
-
-class _WorkflowTaskBase(SQLModel):
-    workflow_id: Optional[int]
-    task_id: Optional[int]
-    order: Optional[int]
-    meta: Optional[Dict[str, Any]]
-    args: Optional[Dict[str, Any]]
-
-
-class WorkflowTaskCreate(_WorkflowTaskBase):
-    task_id: int
-
-
-class WorkflowTaskRead(_WorkflowTaskBase):
-    id: int
-    workflow_id: int
-    task: TaskRead
-
-
-class WorkflowTaskUpdate(_WorkflowTaskBase):
-    args: Optional[Dict[str, Any]]  # type: ignore
-    meta: Optional[Dict[str, Any]]  # type: ignore
-
-    @validator("meta")
-    def check_no_parallelisation_level(cls, m):
-        if "parallelization_level" in m:
-            raise ValueError(
-                "Overriding task parallelization level currently not allowed"
-            )
-        return m
-
-
-class _WorkflowBase(SQLModel):
-    name: str
-    project_id: int
-
-
-class WorkflowRead(_WorkflowBase):
-    id: int
-    task_list: List[WorkflowTaskRead]
-
-
-class WorkflowCreate(_WorkflowBase):
-    pass
-
-
-class WorkflowUpdate(_WorkflowBase):
-    name: Optional[str]  # type: ignore
-    project_id: Optional[int]  # type: ignore
diff --git a/fractal/common/utils.py b/fractal/common/utils.py
deleted file mode 100644
index 8459709a8..000000000
--- a/fractal/common/utils.py
+++ /dev/null
@@ -1,2 +0,0 @@
-def slugify(value: str):
-    return value.lower().replace(" ", "_")
diff --git a/fractal/client/config.py b/fractal/config.py
similarity index 97%
rename from fractal/client/config.py
rename to fractal/config.py
index 2d30ebcbb..b075262e7 100644
--- a/fractal/client/config.py
+++ b/fractal/config.py
@@ -15,7 +15,7 @@
 from dotenv import load_dotenv
 from pydantic import BaseSettings
 
-from .. import __VERSION__
+from . import __VERSION__
 
 
 def fail_getenv(key):
diff --git a/fractal/client/interface.py b/fractal/interface.py
similarity index 100%
rename from fractal/client/interface.py
rename to fractal/interface.py
diff --git a/fractal/client/parser.py b/fractal/parser.py
similarity index 100%
rename from fractal/client/parser.py
rename to fractal/parser.py
diff --git a/fractal/client/response.py b/fractal/response.py
similarity index 100%
rename from fractal/client/response.py
rename to fractal/response.py


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 9bf1d3237..51057313c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -44,7 +44,7 @@ def __clisplit(args: str):
 
 @pytest.fixture
 async def invoke(clisplit):
-    from fractal.client.client import handle
+    from fractal.client import handle
 
     async def __invoke(args: str):
         return await handle(clisplit(args))
@@ -54,7 +54,7 @@ async def __invoke(args: str):
 
 @pytest.fixture
 def clear_task_cache():
-    from fractal.client.config import settings
+    from fractal.config import settings
 
     cache_dir = str(Path(f"{settings.FRACTAL_CACHE_PATH}").expanduser())
     cache_file = f"{cache_dir}/tasks"
diff --git a/tests/fixtures_testserver.py b/tests/fixtures_testserver.py
index 0e245b5cf..a985299aa 100644
--- a/tests/fixtures_testserver.py
+++ b/tests/fixtures_testserver.py
@@ -65,6 +65,9 @@ def run_server():
     # cf. https://stackoverflow.com/a/57816608/283972
     proc = Process(target=run_server, args=(), daemon=True)
     proc.start()
+    import time
+
+    time.sleep(0.2)  # NOTE required to let the server comes up
     logger.debug(environ["FRACTAL_SERVER"])
     yield environ["FRACTAL_SERVER"]
     proc.kill()
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 2991a8de8..58ffbd085 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -2,8 +2,8 @@
 
 import pytest
 
-from fractal.client.authclient import AuthenticationError
-from fractal.client.authclient import AuthToken
+from fractal.authclient import AuthenticationError
+from fractal.authclient import AuthToken
 
 
 async def test_auth_fail(client):