Repository: pyiron/pyiron_base

Issue:
Bug in ProjectPath tests?
Shouldn't [this line](https://github.com/pyiron/pyiron_base/blob/685fa919d25c5e844e4b29548b6291892482f8fe/tests/project/test_projectPath.py#L78) be `root_paths = state.settings.configuration["project_paths"]`?

In `setUpClass` we have `cls.settings_configuration = state.settings.configuration.copy()`, then in `setUp` we have `state.settings.configuration["project_paths"] = [self.current_dir]; state.settings.configuration["project_check_enabled"] = True`, and finally in the test we're setting `root_paths = self.settings_configuration["project_paths"]` and expecting it to be non-empty. Now, this passes on the remote CI, but in `pyironconfig.py` we're explicitly setting `TOP_LEVEL_DIRS` (i.e. `project_paths`), so maybe that allows it to get by even though it's comparing to the wrong thing? On my local machine, where `project_paths = []` and `project_check_enabled = False`, this test fails, exactly because `root_paths == []`.

I get the exact same failure in `test_root_path` for [exactly the same (suspected) reason](https://github.com/pyiron/pyiron_base/blob/685fa919d25c5e844e4b29548b6291892482f8fe/tests/project/test_projectPath.py#L70)

Comments:

Patch:
diff --git a/pyiron_base/jobs/job/path.py b/pyiron_base/jobs/job/path.py
index 0d2efc66..b05f39e5 100644
--- a/pyiron_base/jobs/job/path.py
+++ b/pyiron_base/jobs/job/path.py
@@ -22,19 +22,12 @@ __status__ = "production"
 __date__ = "Sep 1, 2017"
 
 
-class JobPathBase(JobCore):
+class JobPath(JobCore):
     """
     The JobPath class is derived from the JobCore and is used as a lean version of the GenericJob class. Instead of
     loading the full pyiron object the JobPath class only provides access to the HDF5 file, which should be enough
     for most analysis.
 
-    Args:
-        db (DatabaseAccess): database object
-        job_id (int): Job ID - optional, but either a job ID or a database entry db_entry has to be provided.
-        db_entry (dict): database entry {"job":, "subjob":, "projectpath":, "project":, "hamilton":, "hamversion":,
-                                         "status":} and optional entries are {"id":, "masterid":, "parentid":}
-        user (str): current unix/linux/windows user who is running pyiron
-
     Attributes:
 
         .. attribute:: job_name
@@ -109,6 +102,12 @@ class JobPathBase(JobCore):
     """
 
     def __init__(self, job_path):
+        """
+        Load a job from the given path.
+
+        Args:
+            job_path (str): path to the job, must be of the form /path/to/file.h5/job_name
+        """
         job_path_lst = job_path.replace("\\", "/").split(".h5")
         if len(job_path_lst) != 2:
             raise ValueError
@@ -125,9 +124,53 @@ class JobPathBase(JobCore):
             h5_path=h5_path,
             mode="r",
         )
-        super(JobPathBase, self).__init__(
-            project=hdf_project, job_name=job_path_lst[1].split("/")[-1]
-        )
+        super().__init__(project=hdf_project, job_name=job_path_lst[1].split("/")[-1])
+
+    @classmethod
+    def from_job_id(cls, db, job_id):
+        """
+        Load a job path from a database connection and the job id.
+
+        Args:
+            db (DatabaseAccess): database connection
+            job_id (int): Job ID in the database
+        """
+        db_entry = db.get_item_by_id(job_id)
+        if db_entry is None:
+            raise ValueError("job ID {0} does not exist!".format(job_id))
+
+        return cls.from_db_entry(db_entry)
+
+    @classmethod
+    def from_db_entry(cls, db_entry):
+        """
+        Load a job path from a database entry.
+
+        Args:
+            db_entry (dict): database entry {"job":, "subjob":, "projectpath":, "project":, "hamilton":, "hamversion":,
+                                            "status":} and optional entries are {"id":, "masterid":, "parentid":}
+        """
+        hdf5_file = db_entry["subjob"].split("/")[1] + ".h5"
+        job_path = db_entry["projectpath"]
+        if job_path is None:
+            job_path = ""
+        job_path += db_entry["project"] + hdf5_file + db_entry["subjob"]
+        job = cls(job_path=job_path)
+
+        if "hamilton" in db_entry.keys():
+            job.__name__ = db_entry["hamilton"]
+        if "hamversion" in db_entry.keys():
+            job.__version__ = db_entry["hamversion"]
+
+        if "id" in db_entry.keys():
+            job._job_id = db_entry["id"]
+        if "status" in db_entry.keys():
+            job._status = db_entry["status"]
+        if "masterid" in db_entry.keys():
+            job._master_id = db_entry["masterid"]
+        if "parentid" in db_entry.keys():
+            job._parent_id = db_entry["parentid"]
+        return job
 
     @property
     def is_root(self):
@@ -358,117 +401,3 @@ class JobPathBase(JobCore):
             self.project_hdf5._store.close()
         except AttributeError:
             pass
-
-
-class JobPath(JobPathBase):
-    """
-    The JobPath class is derived from the JobCore and is used as a lean version of the GenericJob class. Instead of
-    loading the full pyiron object the JobPath class only provides access to the HDF5 file, which should be enough
-    for most analysis.
-
-    Args:
-        db (DatabaseAccess): database object
-        job_id (int): Job ID - optional, but either a job ID or a database entry db_entry has to be provided.
-        db_entry (dict): database entry {"job":, "subjob":, "projectpath":, "project":, "hamilton":, "hamversion":,
-                                         "status":} and optional entries are {"id":, "masterid":, "parentid":}
-        user (str): current unix/linux/windows user who is running pyiron
-
-    Attributes:
-
-        .. attribute:: job_name
-
-            name of the job, which has to be unique within the project
-
-        .. attribute:: status
-
-            execution status of the job, can be one of the following [initialized, appended, created, submitted, running,
-                                                                      aborted, collect, suspended, refresh, busy, finished]
-
-        .. attribute:: job_id
-
-            unique id to identify the job in the pyiron database
-
-        .. attribute:: parent_id
-
-            job id of the predecessor job - the job which was executed before the current one in the current job series
-
-        .. attribute:: master_id
-
-            job id of the master job - a meta job which groups a series of jobs, which are executed either in parallel or in
-            serial.
-
-        .. attribute:: child_ids
-
-            list of child job ids - only meta jobs have child jobs - jobs which list the meta job as their master
-
-        .. attribute:: project
-
-            Project instance the jobs is located in
-
-        .. attribute:: project_hdf5
-
-            ProjectHDFio instance which points to the HDF5 file the job is stored in
-
-        .. attribute:: job_info_str
-
-            short string to describe the job by it is job_name and job ID - mainly used for logging
-
-        .. attribute:: working_directory
-
-            working directory of the job is executed in - outside the HDF5 file
-
-        .. attribute:: path
-
-            path to the job as a combination of absolute file system path and path within the HDF5 file.
-
-        .. attribute:: is_root
-
-            boolean if the HDF5 object is located at the root level of the HDF5 file
-
-        .. attribute:: is_open
-
-            boolean if the HDF5 file is currently opened - if an active file handler exists
-
-        .. attribute:: is_empty
-
-            boolean if the HDF5 file is empty
-
-        .. attribute:: base_name
-
-            name of the HDF5 file but without any file extension
-
-        .. attribute:: file_path
-
-            directory where the HDF5 file is located
-
-        .. attribute:: h5_path
-
-            path inside the HDF5 file - also stored as absolute path
-    """
-
-    def __init__(self, db, job_id=None, db_entry=None, user=None):
-        if db_entry is None and db is not None:
-            db_entry = db.get_item_by_id(job_id)
-        if db_entry is None:
-            raise ValueError("job ID {0} does not exist!".format(job_id))
-        hdf5_file = db_entry["subjob"].split("/")[1] + ".h5"
-        if db_entry["projectpath"] is not None:
-            job_path = db_entry["projectpath"]
-        else:
-            job_path = ""
-        job_path += db_entry["project"] + hdf5_file + db_entry["subjob"]
-        super(JobPath, self).__init__(job_path=job_path)
-
-        if "hamilton" in db_entry.keys():
-            self.__name__ = db_entry["hamilton"]
-        if "hamversion" in db_entry.keys():
-            self.__version__ = db_entry["hamversion"]
-
-        if "id" in db_entry.keys():
-            self._job_id = db_entry["id"]
-        if "status" in db_entry.keys():
-            self._status = db_entry["status"]
-        if "masterid" in db_entry.keys():
-            self._master_id = db_entry["masterid"]
-        if "parentid" in db_entry.keys():
-            self._parent_id = db_entry["parentid"]
diff --git a/pyiron_base/project/generic.py b/pyiron_base/project/generic.py
index 7b802c34..b85ad7c8 100644
--- a/pyiron_base/project/generic.py
+++ b/pyiron_base/project/generic.py
@@ -8,10 +8,10 @@ The project object is the central import point of pyiron - all other objects can
 import os
 import posixpath
 import shutil
+import stat
 from tqdm.auto import tqdm
 import pandas
 import pint
-import importlib
 import math
 import numpy as np
 
@@ -831,18 +831,17 @@ class Project(ProjectPath, HasGroups):
         Returns:
             GenericJob, JobCore: Either the full GenericJob object or just a reduced JobCore object
         """
-        jobpath = getattr(
-            importlib.import_module("pyiron_base.jobs.job.path"), "JobPath"
-        )
+        from pyiron_base.jobs.job.path import JobPath
+
         if job_id is not None:
-            job = jobpath(db=self.db, job_id=job_id, user=self.user)
+            job = JobPath.from_job_id(db=self.db, job_id=job_id)
             if convert_to_object:
                 job = job.to_object()
                 job.reset_job_id(job_id=job_id)
                 job.set_input_to_read_only()
             return job
         elif db_entry is not None:
-            job = jobpath(db=self.db, db_entry=db_entry)
+            job = JobPath.from_db_entry(db_entry)
             if convert_to_object:
                 job = job.to_object()
                 job.set_input_to_read_only()
@@ -1184,7 +1183,13 @@ class Project(ProjectPath, HasGroups):
                     # dirs and files return values of the iterator are not updated when removing files, so we need to
                     # manually call listdir
                     if len(os.listdir(root)) == 0:
-                        os.rmdir(root)
+                        root = root.rstrip(os.sep)
+                        # the project was symlinked before being deleted
+                        if os.path.islink(root):
+                            os.rmdir(os.readlink(root))
+                            os.remove(root)
+                        else:
+                            os.rmdir(root)
         else:
             raise EnvironmentError("remove() is not available in view_mode!")
 
@@ -1294,9 +1299,9 @@ class Project(ProjectPath, HasGroups):
         Returns:
             GenericJob, JobCore: Either the full GenericJob object or just a reduced JobCore object
         """
-        job = getattr(
-            importlib.import_module("pyiron_base.jobs.job.path"), "JobPathBase"
-        )(job_path=job_path)
+        from pyiron_base.jobs.job.path import JobPath
+
+        job = JobPath(job_path)
         if convert_to_object:
             job = job.to_object()
         job.set_input_to_read_only()
@@ -1676,6 +1681,66 @@ class Project(ProjectPath, HasGroups):
             )
         setattr(cls, name, property(lambda self: tools(self)))
 
+    def symlink(self, target_dir):
+        """
+        Move underlying project folder to target and create a symlink to it.
+
+        The project itself does not change and is not updated in the database.  Instead the project folder is moved into
+        a subdirectory of target_dir with the same name as the project and a symlink is placed in the previous project path
+        pointing to the newly created one.
+
+        If self.path is already a symlink pointing inside target_dir, this method will silently return.
+
+        Args:
+            target_dir (str): new parent folder for the project
+
+        Raises:
+            OSError: when calling this method on non-unix systems
+            RuntimeError: the project path is already a symlink to somewhere else
+            RuntimeError: the project path has submitted or running jobs inside it, wait until after they are finished
+            RuntimeError: target already contains a subdirectory with the project name and it is not empty
+        """
+        target = os.path.join(target_dir, self.name)
+        destination = self.path
+        if destination[-1] == "/":
+            destination = destination[:-1]
+        if stat.S_ISLNK(os.lstat(destination).st_mode):
+            if os.readlink(destination) == target:
+                return
+            raise RuntimeError(
+                "Refusing to symlink and move a project that is already symlinked!"
+            )
+        if os.name != "posix":
+            raise OSError("Symlinking projects is only supported on unix systems!")
+        if len(self.job_table().query('status.isin(["submitted", "running"])')) > 0:
+            raise RuntimeError(
+                "Refusing to symlink and move a project that has submitted or running jobs!"
+            )
+        os.makedirs(target_dir, exist_ok=True)
+        if os.path.exists(target):
+            if len(os.listdir(target)) > 0:
+                raise RuntimeError(
+                    "Refusing to symlink and move a project to non-empty directory!"
+                )
+            else:
+                os.rmdir(target)
+        shutil.move(self.path, target_dir)
+        os.symlink(target, destination)
+
+    def unlink(self):
+        """
+        If the project folder is symlinked somewhere else remove the link and restore the original folder.
+
+        If it is not symlinked, silently return.
+        """
+        path = self.path.rstrip(os.sep)
+        if not stat.S_ISLNK(os.lstat(path).st_mode):
+            return
+
+        target = os.readlink(path)
+        os.unlink(path)
+        shutil.move(target, path)
+
 
 class Creator:
     def __init__(self, project):
diff --git a/pyiron_base/state/install.py b/pyiron_base/state/install.py
index b3b7b634..d220d8f6 100644
--- a/pyiron_base/state/install.py
+++ b/pyiron_base/state/install.py
@@ -141,7 +141,9 @@ def install_pyiron(
         zip_file (str): name of the compressed file
         project_path (str): the location where pyiron is going to store the pyiron projects
         resource_directory (str): the location where the resouces (executables, potentials, ...) for pyiron are stored.
-        giturl_for_zip_file (str): url for the zipped resources file on github
+        giturl_for_zip_file (str/None): url for the zipped resources file on github.
+            (Default points to pyiron's github resource repository. If None, leaves the
+            resources directory *empty*.)
         git_folder_name (str): name of the extracted folder
     """
     _write_config_file(
@@ -149,9 +151,12 @@ def install_pyiron(
         project_path=project_path,
         resource_path=resource_directory,
     )
-    _download_resources(
-        zip_file=zip_file,
-        resource_directory=resource_directory,
-        giturl_for_zip_file=giturl_for_zip_file,
-        git_folder_name=git_folder_name,
-    )
+    if giturl_for_zip_file is not None:
+        _download_resources(
+            zip_file=zip_file,
+            resource_directory=resource_directory,
+            giturl_for_zip_file=giturl_for_zip_file,
+            git_folder_name=git_folder_name,
+        )
+    else:
+        os.mkdir(resource_directory)


Test patch:
diff --git a/tests/project/test_project.py b/tests/project/test_project.py
index 7acf1212..cf533063 100644
--- a/tests/project/test_project.py
+++ b/tests/project/test_project.py
@@ -3,8 +3,9 @@
 # Distributed under the terms of "New BSD License", see the LICENSE file.
 
 import unittest
-from os.path import dirname, join, abspath
-from os import remove
+from os.path import dirname, join, abspath, exists, islink
+import os
+import tempfile
 import pint
 from pyiron_base.project.generic import Project
 from pyiron_base._tests import PyironTestCase, TestWithProject, TestWithFilledProject, ToyJob
@@ -20,7 +21,7 @@ class TestProjectData(PyironTestCase):
     @classmethod
     def tearDownClass(cls):
         try:
-            remove(join(cls.file_location, "pyiron.log"))
+            os.remove(join(cls.file_location, "pyiron.log"))
         except FileNotFoundError:
             pass
 
@@ -127,6 +128,59 @@ class TestProjectOperations(TestWithFilledProject):
         self.assertIn('pyiron_base', df.Module.values)
 
 
+@unittest.skipUnless(os.name=="posix", "symlinking is only available on posix platforms")
+class TestProjectSymlink(TestWithFilledProject):
+    """
+    Test that Project.symlink creates a symlink and unlink removes it again.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.temp = tempfile.TemporaryDirectory()
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        cls.temp.cleanup()
+
+    def test_symlink(self):
+
+        nodes = self.project.list_nodes()
+        groups = self.project.list_groups()
+
+        self.project.symlink(self.temp.name)
+
+        try:
+            self.project.symlink(self.temp.name)
+        except Exception as e:
+            self.fail(f"symlinking twice should have no effect, but raised {e}!")
+
+        with self.assertRaises(RuntimeError, msg="symlinking to another folder should raise an error"):
+            self.project.symlink("asdf")
+
+        path = self.project.path.rstrip(os.sep)
+        self.assertTrue(islink(path), "symlink() did not create a symlink!")
+        self.assertEqual(os.readlink(path), join(self.temp.name, self.project.name),
+                        "symlink() created a wrong symlink!")
+
+        self.assertCountEqual(nodes, self.project.list_nodes(), "not all nodes present after symlink!")
+        self.assertCountEqual(groups, self.project.list_groups(), "not all groups present after symlink!")
+
+        self.project.unlink()
+
+        self.assertTrue(exists(self.project.path), "unlink() did not restore original directory!")
+        self.assertFalse(islink(path), "unlink() did not remove symlink!")
+
+        self.assertCountEqual(nodes, self.project.list_nodes(), "not all nodes present after unlink!")
+        self.assertCountEqual(groups, self.project.list_groups(), "not all groups present after unlink!")
+
+        try:
+            self.project.unlink()
+        except Exception as e:
+            self.fail(f"unlinking twice should have no effect, but raised {e}!")
+
+
 class TestToolRegistration(TestWithProject):
     def setUp(self) -> None:
         self.tools = BaseTools(self.project)
diff --git a/tests/project/test_projectPath.py b/tests/project/test_projectPath.py
index 7a6a752c..00d999be 100644
--- a/tests/project/test_projectPath.py
+++ b/tests/project/test_projectPath.py
@@ -18,13 +18,14 @@ class TestProjectPath(PyironTestCase):
         else:
             cls.current_dir = os.path.dirname(os.path.abspath(__file__))
         cls.settings_configuration = state.settings.configuration.copy()
-        cls.project_path = ProjectPath(path=cls.current_dir)
-        cls.project_path = cls.project_path.open("test_project_path")
 
     def setUp(self) -> None:
-        state.settings.configuration["project_paths"] = [self.current_dir]
+        state.settings.configuration["project_paths"] = [self.current_dir + "/"]
         state.settings.configuration["project_check_enabled"] = True
 
+        self.project_path = ProjectPath(path=self.current_dir)
+        self.project_path = self.project_path.open("test_project_path")
+
     def tearDown(self) -> None:
         state.settings.configuration.update(self.settings_configuration)
 
@@ -67,7 +68,7 @@ class TestProjectPath(PyironTestCase):
         )
 
     def test_root_path(self):
-        root_paths = self.settings_configuration["project_paths"]
+        root_paths = state.settings.configuration["project_paths"]
         self.assertIn(
             self.project_path.root_path,
             root_paths,
@@ -75,7 +76,7 @@ class TestProjectPath(PyironTestCase):
         )
 
     def test_project_path(self):
-        root_paths = self.settings_configuration["project_paths"]
+        root_paths = state.settings.configuration["project_paths"]
         self.assertIn(
             self.current_dir + "/test_project_path/",
             [root_path + self.project_path.project_path for root_path in root_paths],
diff --git a/tests/state/test_install.py b/tests/state/test_install.py
index 733fc675..d4a6e0fd 100644
--- a/tests/state/test_install.py
+++ b/tests/state/test_install.py
@@ -30,6 +30,7 @@ class TestInstall(PyironTestCase):
                 config_file_name=os.path.join(self.execution_path, "config"),
                 resource_directory=os.path.join(self.execution_path, "resources"),
                 project_path=os.path.join(self.execution_path, "project"),
+                giturl_for_zip_file=None,
         )
 
         with open(os.path.join(self.execution_path, "config"), "r") as f: