Repository: pyiron/pyiron_base

Issue:
Features for `pack` and `unpack`
@jan-janssen pointed out in [this PR](https://github.com/pyiron/pyiron_base/pull/1440) that it is now difficult to figure out which tar file contains which project, because now the job table lies in the compressed folder and cannot be easy inspected without being uncompressed. I presume it’s an easy fix so if someone wants to take a look before I come back from the vacation, feel free to do so, otherwise this issue remains for my own reminder.

Hints from issue comments:

Solution patch:
diff --git a/.ci_support/environment-docs.yml b/.ci_support/environment-docs.yml
index 4063f5d2..2a208cab 100644
--- a/.ci_support/environment-docs.yml
+++ b/.ci_support/environment-docs.yml
@@ -22,7 +22,7 @@ dependencies:
 - executorlib =0.0.2
 - pysqa =0.1.21
 - pytables =3.10.1
-- sqlalchemy =2.0.32
+- sqlalchemy =2.0.33
 - tqdm =4.66.5
 - traitlets =5.14.3
 - jupyter-book =1.0.0
diff --git a/.ci_support/environment-mini.yml b/.ci_support/environment-mini.yml
index 65e3cd8f..88f450c1 100644
--- a/.ci_support/environment-mini.yml
+++ b/.ci_support/environment-mini.yml
@@ -13,7 +13,7 @@ dependencies:
 - executorlib =0.0.2
 - pysqa =0.1.21
 - pytables =3.10.1
-- sqlalchemy =2.0.32
+- sqlalchemy =2.0.33
 - tqdm =4.66.5
 - traitlets =5.14.3
 - setuptools
diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml
index d58fc18d..3c07e33a 100644
--- a/.ci_support/environment.yml
+++ b/.ci_support/environment.yml
@@ -20,6 +20,6 @@ dependencies:
 - executorlib =0.0.2
 - pysqa =0.1.21
 - pytables =3.10.1
-- sqlalchemy =2.0.32
+- sqlalchemy =2.0.33
 - tqdm =4.66.5
 - traitlets =5.14.3
diff --git a/.github/workflows/atomistic-notebooks-compat.yml b/.github/workflows/atomistic-notebooks-compat.yml
new file mode 100644
index 00000000..1c7b616b
--- /dev/null
+++ b/.github/workflows/atomistic-notebooks-compat.yml
@@ -0,0 +1,44 @@
+# This workflow is used to check the compatibility with the pyiron_atomistics
+
+name: Compatibility with pyiron_atomistics Notebooks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+
+jobs:
+  build:
+    if: |
+      github.event_name == 'push' ||
+      ( github.event_name == 'pull_request'  && contains(github.event.pull_request.labels.*.name, 'integration' ))
+    
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Merge environment
+      shell: bash -l {0}
+      run: |
+        git clone https://github.com/pyiron/pyiron_atomistics ../pyiron_atomistics
+        cp ../pyiron_atomistics/.ci_support/environment.yml environment.yml
+        tail --lines=+4 ../pyiron_atomistics/.ci_support/environment-notebooks.yml >> environment.yml
+        echo -e "channels:\n  - conda-forge\n" > .condarc
+    - name: Setup Mambaforge
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: "3.12"
+        miniforge-version: latest
+        condarc-file: .condarc
+        environment-file: environment.yml
+    - name: Tests
+      shell: bash -l {0}
+      timeout-minutes: 30
+      run: |
+        pip install versioneer[toml]==0.29
+        cd ../pyiron_atomistics
+        pip install . --no-deps --no-build-isolation
+        cd ../pyiron_base
+        pip install . --no-deps --no-build-isolation
+        cd ../pyiron_atomistics
+        ./.ci_support/build_notebooks.sh
diff --git a/.github/workflows/atomistics-compat.yml b/.github/workflows/atomistics-compat.yml
index e992d33e..a9dec03e 100644
--- a/.github/workflows/atomistics-compat.yml
+++ b/.github/workflows/atomistics-compat.yml
@@ -30,11 +30,9 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Merge environment
+      shell: bash -l {0}
       run: |
         git clone https://github.com/pyiron/pyiron_atomistics ../pyiron_atomistics
-        grep -v "pyiron_base" ../pyiron_atomistics/.ci_support/environment.yml > ../pyiron_atomistics/environment.yml
-        cp .ci_support/environment.yml environment.yml
-        tail --lines=+4 ../pyiron_atomistics/environment.yml >> environment.yml
         echo -e "channels:\n  - conda-forge\n" > .condarc
     - name: Setup Mambaforge
       uses: conda-incubator/setup-miniconda@v3
@@ -42,14 +40,16 @@ jobs:
         python-version: ${{ matrix.python-version }}
         miniforge-version: latest
         condarc-file: .condarc
-        environment-file: environment.yml
+        environment-file: ../pyiron_atomistics/.ci_support/environment.yml
     - name: Tests
       shell: bash -l {0}
       timeout-minutes: 30
       run: |
         pip install versioneer[toml]==0.29
-        pip install . --no-deps --no-build-isolation
         cd ../pyiron_atomistics
         pip install . --no-deps --no-build-isolation
+        cd ../pyiron_base
+        pip install . --no-deps --no-build-isolation
+        cd ../pyiron_atomistics
         python .ci_support/pyironconfig.py
         python -m unittest discover tests/
diff --git a/.github/workflows/contrib-compat.yml b/.github/workflows/contrib-compat.yml
index 797274b6..d3e965d2 100644
--- a/.github/workflows/contrib-compat.yml
+++ b/.github/workflows/contrib-compat.yml
@@ -17,11 +17,9 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Merge environment
+      shell: bash -l {0}
       run: |
         git clone https://github.com/pyiron/pyiron_contrib ../pyiron_contrib
-        grep -v "pyiron_base" ../pyiron_contrib/.ci_support/environment.yml > ../pyiron_contrib/environment.yml
-        cp .ci_support/environment.yml environment.yml
-        tail --lines=+4 ../pyiron_contrib/environment.yml >> environment.yml
         echo -e "channels:\n  - conda-forge\n" > .condarc
     - name: Setup Mambaforge
       uses: conda-incubator/setup-miniconda@v3
@@ -29,7 +27,7 @@ jobs:
         python-version: '3.11'
         miniforge-version: latest
         condarc-file: .condarc
-        environment-file: environment.yml
+        environment-file: ../pyiron_contrib/.ci_support/environment.yml
     - name: Test
       shell: bash -l {0}
       timeout-minutes: 30
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 8c5717fb..8219344e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.2
+    rev: v0.6.3
     hooks:
       - id: ruff
         name: ruff lint
diff --git a/binder/environment.yml b/binder/environment.yml
index afe8e475..94ce06e6 100644
--- a/binder/environment.yml
+++ b/binder/environment.yml
@@ -19,6 +19,6 @@ dependencies:
 - executorlib =0.0.2
 - pysqa =0.1.21
 - pytables =3.10.1
-- sqlalchemy =2.0.32
+- sqlalchemy =2.0.33
 - tqdm =4.66.5
 - traitlets =5.14.3
diff --git a/pyiron_base/interfaces/has_dict.py b/pyiron_base/interfaces/has_dict.py
index 5e619df9..87dd7f40 100644
--- a/pyiron_base/interfaces/has_dict.py
+++ b/pyiron_base/interfaces/has_dict.py
@@ -61,6 +61,95 @@ def create_from_dict(obj_dict):
     return obj
 
 
+def _join_children_dict(children: dict[str, dict[str, Any]]) -> dict[str, Any]:
+    """
+    Given a nested dictionary, flatten the first level.
+
+    >>> d = {'a': {'a1': 3}, 'b': {'b1': 4, 'b2': {'c': 42}}}
+    >>> _join_children_dict(d)
+    {'a/a1': 3, 'b/b1': 4, 'b/b2': {'c': 42}}
+
+    This is intended as a utility function for nested HasDict objects, that
+    to_dict their children and then want to give a flattened dict for
+    writing to ProjectHDFio.write_dict_to_hdf.
+
+    See also :func:`._split_children_dict`.
+    """
+    return {
+        "/".join((k1, k2)): v2 for k1, v1 in children.items() for k2, v2 in v1.items()
+    }
+
+
+def _split_children_dict(obj_dict: dict[str, Any]) -> dict[str, Any | dict[str, Any]]:
+    """
+    Undoes _join_children_dict.
+
+    Classes that use :func:`._join_children_dict` in their `_to_dict`, must
+    call this function in their `_from_dict`.
+    """
+    subs = defaultdict(dict)
+    plain = {}
+    for k, v in obj_dict.items():
+        if "/" not in k:
+            plain[k] = v
+            continue
+        root, k = k.split("/", maxsplit=1)
+        subs[root][k] = v
+    # using update keeps type stability, i.e. we always return a plain dict
+    plain.update(subs)
+    return plain
+
+
+def _from_dict_children(obj_dict: dict) -> dict:
+    """
+    Recurse through `obj_dict` and restore any objects with :class:`~.HasDict`.
+
+    Args:
+        obj_dict (dict): data previously returned from :meth:`.to_dict`
+    """
+
+    def load(inner_dict):
+        # object is a not a dict, so nothing to do
+        if not isinstance(inner_dict, dict):
+            return inner_dict
+        # if object is a dict but doesn't have type information, recurse through it to load any sub dicts that might
+        if not all(k in inner_dict for k in ("NAME", "TYPE", "OBJECT", "DICT_VERSION")):
+            return {k: load(v) for k, v in inner_dict.items()}
+        # object has type info, so just load it
+        return create_from_dict(inner_dict)
+
+    return {k: load(v) for k, v in obj_dict.items()}
+
+
+def _to_dict_children(obj_dict: dict) -> dict:
+    """
+    Call to_dict on any objects in the values that support it.
+
+    Intended as a helper function for recursives object that want to to_dict
+    their nested objects automatically.  It uses :func:`._join_children_dict`
+    for any dictionaries returned from the children.
+
+    The function only goes through the *first* layer of dictionary values and
+    does *not* recurse through nested dictionaries.
+
+    Args:
+        obj_dict (dict): data previously returned from :meth:`._to_dict`
+
+    Returns:
+        obj_dict (dict): new dictionary with the obj_dict of the children
+    """
+    data_dict = {}
+    child_dict = {}
+    for k, v in obj_dict.items():
+        if isinstance(v, HasDict):
+            child_dict[k] = v.to_dict()
+        elif isinstance(v, HasHDF):
+            child_dict[k] = HasDictfromHDF.to_dict(v)
+        else:
+            data_dict[k] = v
+    return data_dict | _join_children_dict(child_dict)
+
+
 class HasDict(ABC):
     """
     Abstract interface to convert objects to dictionaries for storage.
@@ -114,27 +203,18 @@ class HasDict(ABC):
             version (str): version tag written together with the data
         """
 
-        def load(inner_dict):
-            if not isinstance(inner_dict, dict):
-                return inner_dict
-            if not all(
-                k in inner_dict for k in ("NAME", "TYPE", "OBJECT", "DICT_VERSION")
-            ):
-                return {k: load(v) for k, v in inner_dict.items()}
-            return create_from_dict(inner_dict)
-
-        obj_dict = self._split_children_dict(obj_dict)
+        obj_dict = _split_children_dict(obj_dict)
         if version is None:
             version = obj_dict.get("DICT_VERSION", None)
-        self._from_dict({k: load(v) for k, v in obj_dict.items()}, version)
+        self._from_dict(obj_dict, version)
 
     @abstractmethod
     def _from_dict(self, obj_dict: dict, version: str = None):
         """
         Populate the object from the serialized object.
 
-        :meth:`.from_dict` will already recurse through `obj_dict` and deserialize any :class:`.HasDict` data it finds,
-        so implementations do not need to deserialize their children explicitly.
+        Implementations must use :func:`._from_dict_children` if they use
+        `._to_dict_children` in their implementation of :meth:`._to_dict`.
 
         Args:
             obj_dict (dict): data previously returned from :meth:`.to_dict`
@@ -142,7 +222,7 @@ class HasDict(ABC):
         """
         pass
 
-    def to_dict(self):
+    def to_dict(self) -> dict:
         """
         Reduce the object to a dictionary.
 
@@ -150,25 +230,16 @@ class HasDict(ABC):
             dict: serialized state of this object
         """
         type_dict = self._type_to_dict()
-        data_dict = {}
-        child_dict = {}
-        for k, v in self._to_dict().items():
-            if isinstance(v, HasDict):
-                child_dict[k] = v.to_dict()
-            elif isinstance(v, HasHDF):
-                child_dict[k] = HasDictfromHDF.to_dict(v)
-            else:
-                data_dict[k] = v
-        return data_dict | self._join_children_dict(child_dict) | type_dict
+        return self._to_dict() | type_dict
 
     @abstractmethod
-    def _to_dict(self):
+    def _to_dict(self) -> dict:
         """
         Reduce the object to a dictionary.
 
-        :meth:`.to_dict` will find any objects *in the first level* of the returned dictionary and reduce them to
-        dictionaries as well, so implementations do not need to explicitly serialize their children.
-        It will also append the type information obtained from :meth:`._type_to_dict`.
+        Implementations may use :func:`._to_dict_children`, if they
+        automatically want to call `to_dict` on any objects possible from their
+        returned dictionary.
 
         Returns:
             dict: serialized state of this object
@@ -196,44 +267,6 @@ class HasDict(ABC):
             type_dict["VERSION"] = self.__version__
         return type_dict
 
-    @staticmethod
-    def _join_children_dict(children: dict[str, dict[str, Any]]) -> dict[str, Any]:
-        """
-        Given a nested dictionary, flatten the first level.
-
-        >>> d = {'a': {'a1': 3}, 'b': {'b1': 4, 'b2': {'c': 42}}}
-        >>> _join_children_dict(d)
-        {'a/a1': 3, 'b/b1': 4, 'b/b2': {'c': 42}}
-
-        This is intended as a utility function for nested HasDict objects, that
-        to_dict their children and then want to give a flattened dict for
-        writing to ProjectHDFio.write_dict_to_hdf
-        """
-        return {
-            "/".join((k1, k2)): v2
-            for k1, v1 in children.items()
-            for k2, v2 in v1.items()
-        }
-
-    @staticmethod
-    def _split_children_dict(
-        obj_dict: dict[str, Any],
-    ) -> dict[str, Any | dict[str, Any]]:
-        """
-        Undoes _join_children_dict.
-        """
-        subs = defaultdict(dict)
-        plain = {}
-        for k, v in obj_dict.items():
-            if "/" not in k:
-                plain[k] = v
-                continue
-            root, k = k.split("/", maxsplit=1)
-            subs[root][k] = v
-        # using update keeps type stability, i.e. we always return a plain dict
-        plain.update(subs)
-        return plain
-
 
 class HasHDFfromDict(HasHDF, HasDict):
     """
diff --git a/pyiron_base/jobs/job/generic.py b/pyiron_base/jobs/job/generic.py
index a9107f62..1e197fda 100644
--- a/pyiron_base/jobs/job/generic.py
+++ b/pyiron_base/jobs/job/generic.py
@@ -1172,26 +1172,16 @@ class GenericJob(JobCore, HasDict):
             data_dict["files_to_compress"] = self._files_to_compress
         if len(self._files_to_remove) > 0:
             data_dict["files_to_compress"] = self._files_to_remove
+        data_dict["HDF_VERSION"] = self.__version__
         return data_dict
 
     def _from_dict(self, obj_dict, version=None):
         self._type_from_dict(type_dict=obj_dict)
         if "import_directory" in obj_dict.keys():
             self._import_directory = obj_dict["import_directory"]
-        # Backwards compatibility: Previously server and executable were stored
-        # as plain dicts, but now they are dicts with additional info so that
-        # HasDict can load them automatically.
-        # We need to check whether that was possible with the instance check
-        # below and if not, call from_dict ourselves.
-        if isinstance(server := obj_dict["server"], Server):
-            self._server = server
-        else:
-            self._server.from_dict(server)
+        self._server.from_dict(obj_dict["server"])
         if "executable" in obj_dict.keys() and obj_dict["executable"] is not None:
-            if isinstance(executable := obj_dict["executable"], Executable):
-                self._executable = executable
-            else:
-                self.executable.from_dict(executable)
+            self.executable.from_dict(obj_dict["executable"])
         input_dict = obj_dict["input"]
         if "generic_dict" in input_dict.keys():
             generic_dict = input_dict["generic_dict"]
diff --git a/pyiron_base/jobs/job/interactive.py b/pyiron_base/jobs/job/interactive.py
index f8ab53b7..08bfa1e1 100644
--- a/pyiron_base/jobs/job/interactive.py
+++ b/pyiron_base/jobs/job/interactive.py
@@ -389,7 +389,8 @@ class _WithInteractiveOpen:
     def __exit__(self, exc_type, exc_val, exc_tb):
         job_status = self._job.status.string
         job_closed = self._job.interactive_close()
-        self._job.status.string = job_status
+        if job_status in ["aborted"]:
+            self._job.status.string = job_status
         return job_closed
 
     def __getattr__(self, attr):
diff --git a/pyiron_base/project/archiving/import_archive.py b/pyiron_base/project/archiving/import_archive.py
index 775cc5dd..5b8ccab9 100644
--- a/pyiron_base/project/archiving/import_archive.py
+++ b/pyiron_base/project/archiving/import_archive.py
@@ -1,3 +1,4 @@
+import io
 import os
 import posixpath
 import tarfile
@@ -130,3 +131,29 @@ def get_dataframe(origin_path: str, csv_file_name: str = "export.csv") -> "DataF
         if csv_file_name in files:
             return pandas.read_csv(os.path.join(root, csv_file_name), index_col=0)
     raise FileNotFoundError(f"File: {csv_file_name} was not found.")
+
+
+def inspect_csv(tar_path: str, csv_file: str = "export.csv"):
+    """
+    Inspect the csv file inside a tar archive.
+
+    Args:
+        tar_path (str): Path to the tar archive.
+        csv_file (str): Name of the csv file.
+
+    Returns:
+        pandas.DataFrame: Job table.
+    """
+    with tarfile.open(tar_path, mode="r:gz") as tar:
+        for member in tar.getmembers():
+            # Check if the member is a file and ends with the desired csv file name
+            if member.isfile() and member.name.endswith(f"/{csv_file}"):
+                # Extract the file object
+                extracted_file = tar.extractfile(member)
+
+                if extracted_file:
+                    # Read the file content
+                    return pandas.read_csv(
+                        io.StringIO(extracted_file.read().decode("utf-8")), index_col=0
+                    )
+        raise FileNotFoundError(f"File: {csv_file} in {tar_path} was not found.")
diff --git a/pyiron_base/project/generic.py b/pyiron_base/project/generic.py
index 237feaff..2ac6b590 100644
--- a/pyiron_base/project/generic.py
+++ b/pyiron_base/project/generic.py
@@ -1981,6 +1981,21 @@ class Project(ProjectPath, HasGroups):
             df=export_archive.export_database(self.job_table()),
         )
 
+    def unpack_csv(self, tar_path: str):
+        """
+        Import job table from a csv file and copy the content of a project
+        directory from a given path.
+
+        Args:
+            tar_path (str): the relative path of a directory from which the
+                project directory is copied.
+            csv_file (str): the name of the csv file.
+
+        Returns:
+            pandas.DataFrame: job table
+        """
+        return import_archive.inspect_csv(tar_path=tar_path, csv_file="export.csv")
+
     def unpack(self, origin_path, **kwargs):
         """
         by this function, job table is imported from a given csv file,
diff --git a/pyiron_base/storage/datacontainer.py b/pyiron_base/storage/datacontainer.py
index bba83271..7e50ae96 100644
--- a/pyiron_base/storage/datacontainer.py
+++ b/pyiron_base/storage/datacontainer.py
@@ -13,7 +13,12 @@ from collections.abc import Mapping, MutableMapping, Sequence, Set
 import numpy as np
 import pandas
 
-from pyiron_base.interfaces.has_dict import HasDict, HasDictfromHDF
+from pyiron_base.interfaces.has_dict import (
+    HasDict,
+    HasDictfromHDF,
+    _from_dict_children,
+    _to_dict_children,
+)
 from pyiron_base.interfaces.has_groups import HasGroups
 from pyiron_base.interfaces.has_hdf import HasHDF
 from pyiron_base.interfaces.lockable import Lockable, sentinel
@@ -1034,9 +1039,10 @@ class DataContainer(DataContainerBase, HasHDF, HasDict):
         order = list(data)
         data["READ_ONLY"] = self.read_only
         data["KEY_ORDER"] = order
-        return data
+        return _to_dict_children(data)
 
     def _from_dict(self, obj_dict, version=None):
+        obj_dict = _from_dict_children(obj_dict)
         if version == "0.2.0":
             order = obj_dict.pop("KEY_ORDER")
         else:
diff --git a/pyproject.toml b/pyproject.toml
index 6118082b..d08dcd3d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,7 +34,7 @@ dependencies = [
     "pyfileindex==0.0.27",
     "pyiron_snippets==0.1.4",
     "pysqa==0.1.21",
-    "sqlalchemy==2.0.32",
+    "sqlalchemy==2.0.33",
     "tables==3.10.1",
     "tqdm==4.66.5",
     "traitlets==5.14.3",


Test patch:
diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py
index 4ccc93d0..0f037ecf 100644
--- a/tests/unit/archiving/test_import.py
+++ b/tests/unit/archiving/test_import.py
@@ -1,7 +1,7 @@
 import os
 import unittest
 from pyiron_base import Project
-from pandas._testing import assert_frame_equal
+import pandas as pd
 from filecmp import dircmp
 from shutil import rmtree, copytree
 import tarfile
@@ -47,6 +47,11 @@ class TestUnpacking(PyironTestCase):
         super().tearDown()
         self.imp_pr.remove_jobs(recursive=True, silently=True)
 
+    def test_inspect(self):
+        df = self.pr.unpack_csv(self.arch_dir_comp + ".tar.gz")
+        self.assertIsInstance(df, pd.DataFrame)
+        self.assertEqual(len(df), 1)
+
     def test_import_csv(self):
         df_original = self.pr.job_table()
         df_import = self.imp_pr.job_table()
@@ -58,7 +63,7 @@ class TestUnpacking(PyironTestCase):
         df_original.drop("id", inplace=True, axis=1)
         df_import["hamversion"] = float(df_import["hamversion"])
         df_original["hamversion"] = float(df_original["hamversion"])
-        assert_frame_equal(df_original, df_import)
+        pd._testing.assert_frame_equal(df_original, df_import)
 
     def test_import_compressed(self):
         path_original = self.pr.path
@@ -192,6 +197,7 @@ class TestUnpackingBackwardsCompatibility(PyironTestCase):
             dirs_exist_ok=True,
         )
         pr = Project("old_tar")
+        self.assertRaises(FileNotFoundError, pr.unpack_csv, "test_pack.tar.gz")
         pr.unpack(origin_path="test_pack.tar.gz")
         job = pr.load("toy")
         self.assertEqual(job.job_name, "toy")