Repository: glotzerlab/signac

Issue:
Implement a `signac.get_job()` function
**[Original report](https://bitbucket.org/glotzer/signac/issue/108) by Carl Simon Adorf (Bitbucket: [csadorf](https://bitbucket.org/csadorf), GitHub: [csadorf](https://github.com/csadorf)).**

----------------------------------------

The idea is to be able to "get" a job from the local directory. Use cases for example when opening a `signac shell` from within a signac workspace directory.

The main challenge is that the current data model does not associate a job workspace directory with one specific project.

See also related pull request: https://bitbucket.org/glotzer/signac/pull-requests/62


Discussion:
bdice: @csadorf Ready for additional review.
csadorf: I'm going to rebase this branch on develop.
csadorf: > I'm going to rebase this branch on develop.

Just kidding, I merged it.
bdice: @csadorf I have implemented three additional tests for various cases and added more comments to the logic of `get_job`. This addresses all of the cases I can think of. I'm ready to call this done when tests pass.
bdice: @csadorf I addressed your comments. Should be ready to go with your approval.
bdice: @csadorf There are H5Store / numpy related failures on pypy2/3. Doesn't seem related to this PR.
csadorf: > @csadorf There are H5Store / numpy related failures on pypy2/3. Doesn't seem related to this PR.

Not sure how this is possible... 😕  I'll have a look.
csadorf: @bdice I've added an entry to the changelog. I'm ok with updating framework documentation as part of some general overhaul.

Patch:
diff --git a/changelog.txt b/changelog.txt
index 3d171bac..7c598d41 100644
--- a/changelog.txt
+++ b/changelog.txt
@@ -25,6 +25,7 @@ Added
 +++++
 
  - Adds an ``H5Store`` class, useful for storing array-like data with an HDF5 backend. Accessible via ``with job.data:`` or ``with project.data:``.
+ - Adds the ``signac.get_job()`` and the ``signac.Project.get_job()`` functions which allow users to get a job handle by switching into or providing the job's workspace directory.
  - Add automatic cast of numpy arrays to lists when storing them within a `JSONDict`, e.g., a `job.statepoint` or `job.document`.
  - Enable `Collection` class to manage collections stored in gzip files.
  - Enable deleting of `JSONDict` keys through the attribute interface, e.g., `del job.doc.foo`.
diff --git a/signac/__init__.py b/signac/__init__.py
index 1104dd20..ec64fe41 100644
--- a/signac/__init__.py
+++ b/signac/__init__.py
@@ -21,6 +21,7 @@ from .contrib import Project
 from .contrib import TemporaryProject
 from .contrib import get_project
 from .contrib import init_project
+from .contrib import get_job
 from .contrib import fetch
 from .contrib import export_one
 from .contrib import export
@@ -46,7 +47,7 @@ __version__ = '0.9.5'
 
 __all__ = ['__version__', 'contrib', 'db', 'errors', 'warnings', 'sync',
            'cite',
-           'Project', 'TemporaryProject', 'get_project', 'init_project',
+           'Project', 'TemporaryProject', 'get_project', 'init_project', 'get_job',
            'get_database', 'fetch', 'fetch_one',
            'export_one', 'export', 'export_to_mirror',
            'Collection',
diff --git a/signac/__main__.py b/signac/__main__.py
index 0dcd74f8..dcf5fa02 100644
--- a/signac/__main__.py
+++ b/signac/__main__.py
@@ -95,7 +95,7 @@ Total transfer volume:       {stats.volume}
 SHELL_BANNER = """Python {python_version}
 signac {signac_version}
 
-Project:\t{project_id}
+Project:\t{project_id}{job_banner}
 Root:\t\t{root_path}
 Workspace:\t{workspace_path}
 Size:\t\t{size}
@@ -558,6 +558,7 @@ def _main_import_interactive(project, origin, args):
                     python_version=sys.version,
                     signac_version=__version__,
                     project_id=project.get_id(),
+                    job_banner='',
                     root_path=project.root_directory(),
                     workspace_path=project.workspace(),
                     size=len(project),
@@ -958,7 +959,13 @@ def main_shell(args):
             for _id in _jobs:
                 yield project.open_job(id=_id)
 
-        job = _open_job_by_id(project, list(_jobs)[0]) if len(_jobs) == 1 else None
+        if len(_jobs) == 1:
+            job = _open_job_by_id(project, list(_jobs)[0])
+        else:
+            try:
+                job = project.get_job()
+            except LookupError:
+                job = None
 
         local_ns = dict(
             project=project, pr=project,
@@ -988,6 +995,7 @@ def main_shell(args):
                     python_version=sys.version,
                     signac_version=__version__,
                     project_id=project.get_id(),
+                    job_banner='\nJob:\t\t{job._id}'.format(job=job) if job is not None else '',
                     root_path=project.root_directory(),
                     workspace_path=project.workspace(),
                     size=len(project)))
diff --git a/signac/contrib/__init__.py b/signac/contrib/__init__.py
index 8db2b3a8..c886d89d 100644
--- a/signac/contrib/__init__.py
+++ b/signac/contrib/__init__.py
@@ -7,7 +7,7 @@ import logging
 from . import indexing
 from .project import Project
 from .project import TemporaryProject
-from .project import get_project, init_project
+from .project import get_project, init_project, get_job
 from .indexing import BaseCrawler
 from .indexing import RegexFileCrawler
 from .indexing import JSONCrawler
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 __all__ = [
     'indexing',
-    'Project', 'TemporaryProject', 'get_project', 'init_project',
+    'Project', 'TemporaryProject', 'get_project', 'init_project', 'get_job',
     'BaseCrawler', 'RegexFileCrawler', 'JSONCrawler', 'SignacProjectCrawler',
     'MasterCrawler', 'fetch', 'fetch_one', 'fetched',
     'export_one', 'export', 'export_to_mirror', 'export_pymongo',
diff --git a/signac/contrib/job.py b/signac/contrib/job.py
index bebf23d1..beb986fa 100644
--- a/signac/contrib/job.py
+++ b/signac/contrib/job.py
@@ -92,7 +92,7 @@ class Job(object):
         return self._id
 
     def __hash__(self):
-        return hash(self._wd)
+        return hash(os.path.realpath(self._wd))
 
     def __str__(self):
         "Returns the job's id."
diff --git a/signac/contrib/project.py b/signac/contrib/project.py
index 4c4bf49a..e68a52ff 100644
--- a/signac/contrib/project.py
+++ b/signac/contrib/project.py
@@ -1518,7 +1518,7 @@ class Project(object):
         :param search:
             If True, search for project configurations inside and above
             the specified root directory, otherwise only return projects
-            with a root directory identical to the specified root arugment.
+            with a root directory identical to the specified root argument.
         :type search: bool
         :returns: The project handle.
         :raises LookupError: If no project configuration can be found.
@@ -1531,6 +1531,38 @@ class Project(object):
                 "Unable to determine project id for path '{}'.".format(os.path.abspath(root)))
         return cls(config=config)
 
+    @classmethod
+    def get_job(cls, root=None):
+        """Find a Job in or above the current working directory (or provided path).
+
+        :param root: The job root directory.
+            If no root directory is given, the current working directory is
+            assumed to be the job directory.
+        :type root: str
+        :returns: The job handle.
+        :raises LookupError: If this job cannot be found."""
+        if root is None:
+            root = os.getcwd()
+        root = os.path.abspath(root)
+
+        # Ensure the root path exists, which is not guaranteed by the regex match
+        if not os.path.exists(root):
+            raise LookupError("Path does not exist: '{}'.".format(root))
+
+        # Find the last match instance of a job id
+        results = list(re.finditer(JOB_ID_REGEX, root))
+        if len(results) == 0:
+            raise LookupError("Could not find a job id in path '{}'.".format(root))
+        match = results[-1]
+        job_id = match.group(0)
+        job_root = root[:match.end()]
+
+        # Find a project *above* the root directory (avoid finding nested projects)
+        project = cls.get_project(os.path.join(job_root, os.pardir))
+
+        # Return the matched job id from the found project
+        return project.open_job(id=job_id)
+
 
 @contextmanager
 def TemporaryProject(name=None, cls=None, **kwargs):
@@ -1836,6 +1868,28 @@ def get_project(root=None, search=True):
         with a root directory identical to the specified root arugment.
     :type search: bool
     :returns: The project handle.
+    :rtype: :py:class:`~.Project`
     :raises LookupError: If no project configuration can be found.
     """
     return Project.get_project(root=root, search=search)
+
+
+def get_job(root=None):
+    """Find a Job in or above the current working directory (or provided path).
+
+    :param root: The job root directory.
+        If no root directory is given, the current working directory is
+        assumed to be within the current job workspace directory.
+    :type root: str
+    :returns: The job handle.
+    :raises LookupError: If this job cannot be found.
+
+    For example, when the current directory is a job workspace directory:
+
+    .. code-block:: python
+
+        >>> signac.get_job()
+        signac.contrib.job.Job(project=..., statepoint={...})
+
+    """
+    return Project.get_job(root=root)


Test patch:
diff --git a/tests/test_project.py b/tests/test_project.py
index 6ef330b7..352f0218 100644
--- a/tests/test_project.py
+++ b/tests/test_project.py
@@ -1765,6 +1765,20 @@ class ProjectInitTest(unittest.TestCase):
         self.assertEqual(project.workspace(), os.path.join(root, 'workspace'))
         self.assertEqual(project.root_directory(), root)
 
+    def test_get_project_non_local(self):
+        root = self._tmp_dir.name
+        subdir = os.path.join(root, 'subdir')
+        os.mkdir(subdir)
+        project = signac.init_project(root=root, name='testproject')
+        self.assertEqual(project, project.get_project(root=root))
+        self.assertEqual(project, signac.get_project(root=root))
+        with self.assertRaises(LookupError):
+            self.assertEqual(project, project.get_project(root=subdir, search=False))
+        with self.assertRaises(LookupError):
+            self.assertEqual(project, signac.get_project(root=subdir, search=False))
+        self.assertEqual(project, project.get_project(root=subdir, search=True))
+        self.assertEqual(project, signac.get_project(root=subdir, search=True))
+
     def test_init(self):
         root = self._tmp_dir.name
         with self.assertRaises(LookupError):
@@ -1827,19 +1841,94 @@ class ProjectInitTest(unittest.TestCase):
         finally:
             os.chdir(cwd)
 
-    def test_get_project_non_local(self):
+    def test_get_job_valid_workspace(self):
+        # Test case: The root-path is the job workspace path.
         root = self._tmp_dir.name
-        subdir = os.path.join(root, 'subdir')
-        os.mkdir(subdir)
-        project = signac.init_project(root=root, name='testproject')
-        self.assertEqual(project, project.get_project(root=root))
-        self.assertEqual(project, signac.get_project(root=root))
-        with self.assertRaises(LookupError):
-            self.assertEqual(project, project.get_project(root=subdir, search=False))
-        with self.assertRaises(LookupError):
-            self.assertEqual(project, signac.get_project(root=subdir, search=False))
-        self.assertEqual(project, project.get_project(root=subdir, search=True))
-        self.assertEqual(project, signac.get_project(root=subdir, search=True))
+        project = signac.init_project(name='testproject', root=root)
+        job = project.open_job({'a': 1})
+        job.init()
+        with job:
+            # The context manager enters the working directory of the job
+            self.assertEqual(project.get_job(), job)
+            self.assertEqual(signac.get_job(), job)
+
+    def test_get_job_invalid_workspace(self):
+        # Test case: The root-path is not the job workspace path.
+        root = self._tmp_dir.name
+        project = signac.init_project(name='testproject', root=root)
+        job = project.open_job({'a': 1})
+        job.init()
+        # We shouldn't be able to find a job while in the workspace directory,
+        # since no signac_statepoint.json exists.
+        cwd = os.getcwd()
+        try:
+            os.chdir(project.workspace())
+            with self.assertRaises(LookupError):
+                project.get_job()
+            with self.assertRaises(LookupError):
+                signac.get_job()
+        finally:
+            os.chdir(cwd)
+
+    def test_get_job_nested_project(self):
+        # Test case: The job workspace dir is also a project root dir.
+        root = self._tmp_dir.name
+        project = signac.init_project(name='testproject', root=root)
+        job = project.open_job({'a': 1})
+        job.init()
+        with job:
+            nestedproject = signac.init_project('nestedproject')
+            nestedproject.open_job({'b': 2}).init()
+            self.assertEqual(project.get_job(), job)
+            self.assertEqual(signac.get_job(), job)
+
+    def test_get_job_subdir(self):
+        # Test case: Get a job from a sub-directory of the job workspace dir.
+        root = self._tmp_dir.name
+        project = signac.init_project(name='testproject', root=root)
+        job = project.open_job({'a': 1})
+        job.init()
+        with job:
+            os.mkdir('test_subdir')
+            self.assertEqual(project.get_job('test_subdir'), job)
+            self.assertEqual(signac.get_job('test_subdir'), job)
+        self.assertEqual(project.get_job(job.fn('test_subdir')), job)
+        self.assertEqual(signac.get_job(job.fn('test_subdir')), job)
+
+    def test_get_job_nested_project_subdir(self):
+        # Test case: Get a job from a sub-directory of the job workspace dir
+        # when the job workspace is also a project root dir
+        root = self._tmp_dir.name
+        project = signac.init_project(name='testproject', root=root)
+        job = project.open_job({'a': 1})
+        job.init()
+        with job:
+            nestedproject = signac.init_project('nestedproject')
+            nestedproject.open_job({'b': 2}).init()
+            os.mkdir('test_subdir')
+            self.assertEqual(project.get_job('test_subdir'), job)
+            self.assertEqual(signac.get_job('test_subdir'), job)
+        self.assertEqual(project.get_job(job.fn('test_subdir')), job)
+        self.assertEqual(signac.get_job(job.fn('test_subdir')), job)
+
+    def test_get_job_symlink_other_project(self):
+        # Test case: Get a job from a symlink in another project workspace
+        root = self._tmp_dir.name
+        project_a_dir = os.path.join(root, 'project_a')
+        project_b_dir = os.path.join(root, 'project_b')
+        os.mkdir(project_a_dir)
+        os.mkdir(project_b_dir)
+        project_a = signac.init_project(name='project_a', root=project_a_dir)
+        project_b = signac.init_project(name='project_b', root=project_b_dir)
+        job_a = project_a.open_job({'a': 1})
+        job_a.init()
+        job_b = project_b.open_job({'b': 1})
+        job_b.init()
+        symlink_path = os.path.join(project_b.workspace(), job_a._id)
+        os.symlink(job_a.ws, symlink_path)
+        self.assertEqual(project_a.get_job(symlink_path), job_a)
+        self.assertEqual(project_b.get_job(symlink_path), job_a)
+        self.assertEqual(signac.get_job(symlink_path), job_a)
 
 
 class ProjectPicklingTest(BaseProjectTest):