Repository: pyiron/pyiron_base

Issue:
Overwriting `None` by atomistic structure in project data raises error
The following lines work perfectly:
```python
from pyiron_atomistics import Project
pr = Project('TEST')
pr.data.something = None
pr.data.write()
pr.data.something = [1, 2, 3]
pr.data.write()
pr = Project('TEST')
print(pr.data.something)
```

Output:
```python
>>> [1, 2, 3]
```

However, the following lines raise an error:

```python
from pyiron_atomistics import Project
pr = Project('TEST')
pr.data.something = None
pr.data.write()
pr.data.something = pr.create.structure.bulk('Fe')
pr.data.write()
```

```python-traceback
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/tmp/ipykernel_2091/4008969041.py in <cell line: 7>()
      5 pr.data.something = pr.create.structure.bulk('Fe')
      6 # pr.data.something = [1, 2, 3]
----> 7 pr.data.write()

~/dev_sam/pyiron_base/pyiron_base/project/data.py in write(self)
     58     def write(self):
     59         """Write data to project-level storage."""
---> 60         self.to_hdf(ProjectHDFio(self._project, file_name="project_data"))

~/dev_sam/pyiron_base/pyiron_base/interfaces/has_hdf.py in to_hdf(self, hdf, group_name)
    212             if len(hdf.list_dirs()) > 0 and group_name is None:
    213                 raise ValueError("HDF group must be empty when group_name is not set.")
--> 214             self._to_hdf(hdf)
    215             self._store_type_to_hdf(hdf)
    216 

~/dev_sam/pyiron_base/pyiron_base/storage/datacontainer.py in _to_hdf(self, hdf)
    801                 v, (pandas.DataFrame, pandas.Series)
    802             ):
--> 803                 v.to_hdf(hdf=hdf, group_name=k)
    804             else:
    805                 # if the value doesn't know how to serialize itself, assume

~/dev_sam/pyiron_atomistics/pyiron_atomistics/atomistics/structure/atoms.py in to_hdf(self, hdf, group_name)
    460         with hdf.open(group_name) as hdf_structure:
    461             # time_start = time.time()
--> 462             hdf_structure["TYPE"] = str(type(self))
    463             for el in self.species:
    464                 if isinstance(el.tags, dict):

~/dev_sam/pyiron_base/pyiron_base/storage/hdfio.py in __setitem__(self, key, value)
    258         elif isinstance(value, tuple):
    259             value = list(value)
--> 260         h5io.write_hdf5(
    261             self.file_name,
    262             value,

/u/system/SLES12/soft/pyiron/dev/anaconda3/lib/python3.8/site-packages/h5io/_h5io.py in write_hdf5(fname, data, overwrite, compression, title, slash, use_json)
    109             del fid[title]
    110         cleanup_data = []
--> 111         _triage_write(title, data, fid, comp_kw, str(type(data)),
    112                       cleanup_data, slash=slash, title=title,
    113                       use_json=use_json)

/u/system/SLES12/soft/pyiron/dev/anaconda3/lib/python3.8/site-packages/h5io/_h5io.py in _triage_write(key, value, root, comp_kw, where, cleanup_data, slash, title, use_json)
    182             value = np.frombuffer(value.encode('ASCII'), np.uint8)
    183             title = 'ascii'
--> 184         _create_titled_dataset(root, key, title, value, comp_kw)
    185     elif isinstance(value, np.ndarray):
    186         if not (value.dtype == np.dtype('object') and

/u/system/SLES12/soft/pyiron/dev/anaconda3/lib/python3.8/site-packages/h5io/_h5io.py in _create_titled_dataset(root, key, title, data, comp_kw)
     46     """Helper to create a titled dataset in h5py"""
     47     comp_kw = {} if comp_kw is None else comp_kw
---> 48     out = root.create_dataset(key, data=data, **comp_kw)
     49     out.attrs['TITLE'] = title
     50     return out

/u/system/SLES12/soft/pyiron/dev/anaconda3/lib/python3.8/site-packages/h5py/_hl/group.py in create_dataset(self, name, shape, dtype, data, **kwds)
    157                 if b'/' in name.lstrip(b'/'):
    158                     parent_path, name = name.rsplit(b'/', 1)
--> 159                     group = self.require_group(parent_path)
    160 
    161             dsid = dataset.make_new_dset(group, shape, dtype, data, name, **kwds)

/u/system/SLES12/soft/pyiron/dev/anaconda3/lib/python3.8/site-packages/h5py/_hl/group.py in require_group(self, name)
    314             grp = self[name]
    315             if not isinstance(grp, Group):
--> 316                 raise TypeError("Incompatible object (%s) already exists" % grp.__class__.__name__)
    317             return grp
    318 

TypeError: Incompatible object (Dataset) already exists
```

Not sure if it's a feature or a bug.

EDIT:
ghost edit by @niklassiemer adding the traceback formatting (python-traceback)

Comments:

Patch:
diff --git a/pyiron_base/interfaces/has_hdf.py b/pyiron_base/interfaces/has_hdf.py
index 94e7b268..656d003c 100644
--- a/pyiron_base/interfaces/has_hdf.py
+++ b/pyiron_base/interfaces/has_hdf.py
@@ -23,6 +23,8 @@ class _WithHDF:
     __slots__ = ("_hdf", "_group_name")
 
     def __init__(self, hdf, group_name=None):
+        if group_name in hdf.list_nodes():
+            raise ValueError(f"{group_name} is a node and not a group!")
         self._hdf = hdf
         self._group_name = group_name
 
@@ -209,7 +211,10 @@ class HasHDF(ABC):
             group_name if group_name is not None else self._get_hdf_group_name()
         )
         with _WithHDF(hdf, group_name) as hdf:
-            if len(hdf.list_dirs()) > 0 and group_name is None:
+            if (
+                group_name is None
+                and (len(hdf.list_nodes()) > 0 or len(hdf.list_dirs())) > 0
+            ):
                 raise ValueError("HDF group must be empty when group_name is not set.")
             self._to_hdf(hdf)
             self._store_type_to_hdf(hdf)
diff --git a/pyiron_base/interfaces/object.py b/pyiron_base/interfaces/object.py
index 078b66c2..5f48e42e 100644
--- a/pyiron_base/interfaces/object.py
+++ b/pyiron_base/interfaces/object.py
@@ -27,15 +27,27 @@ __date__ = "Mar 23, 2021"
 class HasStorage(HasHDF, ABC):
     """
     A base class for objects that use HDF5 data serialization via the `DataContainer` class.
+
+    Unless you know what you are doing sub classes should pass the `group_name` argument to :meth:`~.__init__` or
+    override :meth:`~.get_hdf_group_name()` to force a default name for the HDF group the object should write itself to.
     """
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, group_name=None, **kwargs):
+        """
+
+        Args:
+            group_name (str): default name of the HDF group where the whole object should be written to.
+        """
         self._storage = DataContainer(table_name="storage")
+        self._group_name = group_name
 
     @property
     def storage(self) -> DataContainer:
         return self._storage
 
+    def _get_hdf_group_name(self) -> str:
+        return self._group_name
+
     def _to_hdf(self, hdf: ProjectHDFio):
         self.storage.to_hdf(hdf=hdf)
 
diff --git a/pyiron_base/jobs/job/template.py b/pyiron_base/jobs/job/template.py
index fa73ad58..576e14f9 100644
--- a/pyiron_base/jobs/job/template.py
+++ b/pyiron_base/jobs/job/template.py
@@ -24,7 +24,7 @@ __date__ = "May 15, 2020"
 class TemplateJob(GenericJob, HasStorage):
     def __init__(self, project, job_name):
         GenericJob.__init__(self, project, job_name)
-        HasStorage.__init__(self)
+        HasStorage.__init__(self, group_name="")
         self.storage.create_group("input")
         self.storage.create_group("output")
 
@@ -38,11 +38,11 @@ class TemplateJob(GenericJob, HasStorage):
 
     def to_hdf(self, hdf=None, group_name=None):
         GenericJob.to_hdf(self, hdf=hdf, group_name=group_name)
-        HasStorage.to_hdf(self, hdf=self.project_hdf5, group_name="")
+        HasStorage.to_hdf(self, hdf=self.project_hdf5)
 
     def from_hdf(self, hdf=None, group_name=None):
         GenericJob.from_hdf(self, hdf=hdf, group_name=group_name)
-        HasStorage.from_hdf(self, hdf=self.project_hdf5, group_name="")
+        HasStorage.from_hdf(self, hdf=self.project_hdf5)
 
 
 class PythonTemplateJob(TemplateJob):
diff --git a/pyiron_base/storage/datacontainer.py b/pyiron_base/storage/datacontainer.py
index b33fbeca..21004574 100644
--- a/pyiron_base/storage/datacontainer.py
+++ b/pyiron_base/storage/datacontainer.py
@@ -790,16 +790,22 @@ class DataContainer(MutableMapping, HasGroups, HasHDF):
 
     def _to_hdf(self, hdf):
         hdf["READ_ONLY"] = self.read_only
+        written_keys = _internal_hdf_nodes.copy()
         for i, (k, v) in enumerate(self.items()):
             if isinstance(k, str) and "__index_" in k:
                 raise ValueError("Key {} clashes with internal use!".format(k))
 
             k = "{}__index_{}".format(k if isinstance(k, str) else "", i)
+            written_keys.append(k)
 
             # pandas objects also have a to_hdf method that is entirely unrelated to ours
             if hasattr(v, "to_hdf") and not isinstance(
                 v, (pandas.DataFrame, pandas.Series)
             ):
+                # if v will be written as a group, but a node of the same name k exists already in the file, h5py will
+                # complain, so delete it first
+                if k in hdf.list_nodes():
+                    del hdf[k]
                 v.to_hdf(hdf=hdf, group_name=k)
             else:
                 # if the value doesn't know how to serialize itself, assume
@@ -811,6 +817,12 @@ class DataContainer(MutableMapping, HasGroups, HasHDF):
                         "Error saving {} (key {}): DataContainer doesn't support saving elements "
                         'of type "{}" to HDF!'.format(v, k, type(v))
                     ) from None
+        for n in hdf.list_nodes():
+            if n not in written_keys:
+                del hdf[n]
+        for g in hdf.list_groups():
+            if g not in written_keys:
+                del hdf[g]
 
     def _from_hdf(self, hdf, version=None):
         self.clear()
@@ -916,7 +928,7 @@ class DataContainer(MutableMapping, HasGroups, HasHDF):
         if not self._lazy and not recursive:
             return
 
-        # values are loaded from HDF once they are accessed via __getitem__, which is implicetly called by values()
+        # values are loaded from HDF once they are accessed via __getitem__, which is implicitly called by values()
         for v in self.values():
             if recursive and isinstance(v, DataContainer):
                 v._force_load()
diff --git a/pyiron_base/storage/hdfio.py b/pyiron_base/storage/hdfio.py
index 5e787c3c..198dc767 100644
--- a/pyiron_base/storage/hdfio.py
+++ b/pyiron_base/storage/hdfio.py
@@ -260,7 +260,7 @@ class FileHDFio(HasGroups, MutableMapping):
         h5io.write_hdf5(
             self.file_name,
             value,
-            title=posixpath.join(self.h5_path, key),
+            title=self._get_h5_path(key),
             overwrite="update",
             use_json=use_json,
         )
@@ -275,7 +275,7 @@ class FileHDFio(HasGroups, MutableMapping):
         if self.file_exists:
             try:
                 with open_hdf5(self.file_name, mode="a") as store:
-                    del store[key]
+                    del store[self._get_h5_path(key)]
             except (AttributeError, KeyError):
                 pass
 
@@ -531,7 +531,7 @@ class FileHDFio(HasGroups, MutableMapping):
         Returns:
             FileHDFio: FileHDFio object pointing to the new group
         """
-        full_name = posixpath.join(self.h5_path, name)
+        full_name = self._get_h5_path(name)
         with open_hdf5(self.file_name, mode="a") as h:
             try:
                 h.create_group(full_name, track_order=track_order)
@@ -570,7 +570,7 @@ class FileHDFio(HasGroups, MutableMapping):
         if h5_rel_path.strip() == ".":
             h5_rel_path = ""
         if h5_rel_path.strip() != "":
-            new_h5_path.h5_path = posixpath.join(new_h5_path.h5_path, h5_rel_path)
+            new_h5_path.h5_path = self._get_h5_path(h5_rel_path)
         new_h5_path.history.append(h5_rel_path)
 
         return new_h5_path


Test patch:
diff --git a/tests/generic/test_datacontainer.py b/tests/generic/test_datacontainer.py
index 98f66bad..60ef96f9 100644
--- a/tests/generic/test_datacontainer.py
+++ b/tests/generic/test_datacontainer.py
@@ -39,7 +39,13 @@ class TestDataContainer(TestWithCleanProject):
             ]}
         ], table_name="input")
         cls.pl["tail"] = DataContainer([2, 4, 8])
-        cls.hdf = cls.project.create_hdf(cls.project.path, "test")
+
+    def setUp(self):
+        self.hdf = self.project.create_hdf(self.project.path, "test")
+
+    def tearDown(self):
+        self.hdf.remove_file()
+        self.hdf = None
 
     # Init tests
     def test_init_none(self):
@@ -397,7 +403,7 @@ class TestDataContainer(TestWithCleanProject):
         self.assertTrue("READ_ONLY" in self.hdf["read_only_f"].list_nodes(), "read-only parameter not saved in HDF")
         self.assertEqual(
             self.pl.read_only,
-            self.hdf[self.pl.table_name]["READ_ONLY"],
+            self.hdf["read_only_f"]["READ_ONLY"],
             "read-only parameter not correctly written to HDF"
         )
 
@@ -460,6 +466,7 @@ class TestDataContainer(TestWithCleanProject):
     def test_hdf_empty_group(self):
         """Writing a list without table_name or group_name should only work if the HDF group is empty."""
         l = DataContainer([1, 2, 3])
+        self.hdf["dummy"] = True
         with self.assertRaises(ValueError, msg="No exception when writing to full hdf group."):
             l.to_hdf(self.hdf)
         h = self.hdf.create_group("empty_group")
@@ -630,6 +637,53 @@ class TestDataContainer(TestWithCleanProject):
         self.assertTrue(not isinstance(ll._store[0], HDFStub),
                         "Loaded value not stored back into container!")
 
+    def test_overwrite_with_group(self):
+        """Writing to HDF second time after replacing a node by a group should not raise an error."""
+        d = DataContainer({"test": 42})
+        d.to_hdf(hdf=self.hdf, group_name="overwrite")
+        del d.test
+        d.create_group("test")
+        d.test.foo = 42
+        try:
+            d.to_hdf(hdf=self.hdf, group_name="overwrite")
+        except Exception as e:
+            self.fail(f"to_hdf raised \"{e}\"!")
+
+    def test_overwrite_with_node(self):
+        """Writing to HDF second time after replacing a group by a node should not raise an error."""
+        d = DataContainer({"test": {"foo": 42}})
+        d.to_hdf(hdf=self.hdf, group_name="overwrite")
+        del d.test
+        d.create_group("test")
+        d.test = 42
+        try:
+            d.to_hdf(hdf=self.hdf, group_name="overwrite")
+        except Exception as e:
+            self.fail(f"to_hdf raised \"{e}\"!")
+
+    def test_overwrite_no_dangling_items(self):
+        """Writing to HDF a second time should leave only items in HDF that are currently in the container."""
+        d = self.pl.copy()
+        d.to_hdf(self.hdf)
+        del d[len(d) - 1]
+        d.to_hdf(self.hdf)
+        items = [k for k in self.hdf[d.table_name].list_nodes() if "__index_" in k] \
+              + [k for k in self.hdf[d.table_name].list_groups() if "__index_" in k]
+        self.assertEqual(len(d), len(items),
+                         "Number of items in HDF does not match length of container!")
+
+    def test_overwrite_ordering(self):
+        """Writing to HDF a second time with different item order should not leave other items in the HDF."""
+        d = self.pl.copy()
+        d.to_hdf(self.hdf)
+        d = DataContainer(list(reversed(list(d.values()))),
+                          table_name=d.table_name)
+        d.to_hdf(self.hdf)
+        items = [k for k in self.hdf[d.table_name].list_nodes() if "__index_" in k] \
+              + [k for k in self.hdf[d.table_name].list_groups() if "__index_" in k]
+        self.assertEqual(len(d), len(items),
+                         "Number of items in HDF does not match length of container!")
+
 
 class TestInputList(PyironTestCase):
 
diff --git a/tests/generic/test_fileHDFio.py b/tests/generic/test_fileHDFio.py
index a35aa4f5..f9a08627 100644
--- a/tests/generic/test_fileHDFio.py
+++ b/tests/generic/test_fileHDFio.py
@@ -293,6 +293,14 @@ class TestFileHDFio(PyironTestCase):
         self.to_be_removed_hdf.remove_file()
         self.assertFalse(os.path.isfile(path))
 
+    def test_delitem(self):
+        """After deleting an entry, it should not be accessible anymore."""
+        with self.full_hdf5.open("content") as opened_hdf:
+            opened_hdf["dummy"] = 42
+            del opened_hdf["dummy"]
+            self.assertNotIn("dummy", opened_hdf.list_nodes(), msg="Entry still in HDF after del!")
+
+
     def test_get_from_table(self):
         pass