Repository: esi-neuroscience/syncopy

Issue:
Syncopy does not keep track of its temp storage 
**Describe the bug**
Syncopy does not correctly compute the size of its temporary storage directory. 

**To Reproduce**
```python
import syncopy as spy

Syncopy 2022.12

See https://syncopy.org for the online documentation.
For bug reports etc. please send an email to syncopy@esi-frankfurt.de

Logging to log directory '/cs/home/kajald/.spy/logs'.
Temporary storage directory set to '/cs/home/kajald/.spy/tmp_storage'

from syncopy.datatype.util import setup_storage
setup_storage()
(1.9789702035486698, 8)

```
However, 

```bash
du -h --max-depth=1 /cs/home/kajald/.spy/tmp_storage/
1.5G	/cs/home/kajald/.spy/tmp_storage/spy_fe54_7bb8e9a9
14G	/cs/home/kajald/.spy/tmp_storage/spy_ca09_b73e54ff
512	/cs/home/kajald/.spy/tmp_storage/spy_fe54_bf55d6f2
281G	/cs/home/kajald/.spy/tmp_storage/spy_fe54_6d6399c5
512	/cs/home/kajald/.spy/tmp_storage/spy_fe54_deffa14a
512	/cs/home/kajald/.spy/tmp_storage/spy_fe54_41496fd4
512	/cs/home/kajald/.spy/tmp_storage/spy_fe54_194bcf7e
301G	/cs/home/kajald/.spy/tmp_storage/
```

**Expected behavior**
Print a warning message if the size of the storage directory exceeds the pre-set limit (of 10GB, I think). 

**System Profile:**
 - OS: Linux (RHEL 8.2)



Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 86c2a0bc..ca535c2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,8 @@ All notable changes to this project will be documented in this file.
 - fix bug #394 'Copying a spy.StructDict returns a dict'.
 - serializable `.cfg` #392
 - single trial cross-corr bug #446
+- fix bug #457, Syncopy does not warn about temp storage dir size exceeding reporting threshold at startup
+
 
 ## [2022.12]
 
diff --git a/syncopy/__init__.py b/syncopy/__init__.py
index 8becb36e..8f27f3a0 100644
--- a/syncopy/__init__.py
+++ b/syncopy/__init__.py
@@ -36,13 +36,14 @@ except PackageNotFoundError:
 
 # --- Greeting ---
 
-def startup_print_once(message):
-    """Print message once: do not spam message n times during all n worker imports."""
+def startup_print_once(message, force=False):
+    """Print message once: do not spam message n times during all n worker imports.
+    """
     try:
         dd.get_client()
     except ValueError:
         silence_file = os.path.join(os.path.expanduser("~"), ".spy", "silentstartup")
-        if os.getenv("SPYSILENTSTARTUP") is None and not os.path.isfile(silence_file):
+        if force or (os.getenv("SPYSILENTSTARTUP") is None and not os.path.isfile(silence_file)):
             print(message)
 
 
@@ -141,23 +142,28 @@ from .statistics import *
 from .plotting import *
 from .preproc import *
 
-from .datatype.util import setup_storage
+from .datatype.util import setup_storage, get_dir_size
 storage_tmpdir_size_gb, storage_tmpdir_numfiles = setup_storage()  # Creates the storage dir if needed and computes size and number of files in there if any.
+spydir_size_gb, spydir_numfiles = get_dir_size(spydir, out="GB")
 
 from .shared.log import setup_logging
 __logdir__ = None  # Gets set in setup_logging() call below.
 setup_logging(spydir=spydir, session=__sessionid__)  # Sets __logdir__.
 startup_print_once(f"Logging to log directory '{__logdir__}'.\nTemporary storage directory set to '{__storage__}'.\n")
 
+storage_msg = (
+        "\nSyncopy <core> WARNING: {folder_desc}:s '{tmpdir:s}' "
+        + "contains {nfs:d} files taking up a total of {sze:4.2f} GB on disk. \n"
+        + "Please run `spy.cleanup()` and/or manually free up disk space."
+    )
 if storage_tmpdir_size_gb > __storagelimit__:
-            msg = (
-                "\nSyncopy <core> WARNING: Temporary storage folder {tmpdir:s} "
-                + "contains {nfs:d} files taking up a total of {sze:4.2f} GB on disk. \n"
-                + "Consider running `spy.cleanup()` to free up disk space."
-            )
-            msg_formatted = msg.format(tmpdir=__storage__, nfs=storage_tmpdir_numfiles, sze=storage_tmpdir_size_gb)
-            startup_print_once(msg_formatted)
-
+    msg_formatted = storage_msg.format(folder_desc="Temporary storage folder", tmpdir=__storage__, nfs=storage_tmpdir_numfiles, sze=storage_tmpdir_size_gb)
+    startup_print_once(msg_formatted, force=True)
+else:
+    # We also check the size of the whole Syncopy cfg folder, as older Syncopy versions placed files directly into it.
+    if spydir_size_gb > __storagelimit__:
+        msg_formatted = storage_msg.format(folder_desc="User config folder", tmpdir=spydir, nfs=spydir_numfiles, sze=spydir_size_gb)
+        startup_print_once(msg_formatted, force=True)
 
 # Override default traceback (differentiate b/w Jupyter/iPython and regular Python)
 from .shared.errors import SPYExceptionHandler
diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py
index c6e5b1e8..6f05555a 100644
--- a/syncopy/datatype/base_data.py
+++ b/syncopy/datatype/base_data.py
@@ -1255,7 +1255,7 @@ class BaseData(ABC):
                     prop.file.close()
                 # can happen if the file was deleted elsewhere
                 # or we exit un-gracefully from some undefined state
-                except (ValueError, ImportError):
+                except (ValueError, ImportError, TypeError):
                     pass
 
         # remove from file system
diff --git a/syncopy/datatype/util.py b/syncopy/datatype/util.py
index 6411737e..dac7dc99 100644
--- a/syncopy/datatype/util.py
+++ b/syncopy/datatype/util.py
@@ -58,9 +58,35 @@ class TrialIndexer:
         return "{} element iterable".format(self._len)
 
 
-def setup_storage():
+def get_dir_size(start_path = '.', out="byte"):
     """
-    Create temporary storage dir and report on its size.
+    Compute size of all files in directory (and its subdirectories), in bytes or GB.
+    """
+    total_size_bytes = 0
+    num_files = 0
+    for dirpath, _, filenames in os.walk(start_path):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            # skip if it is symbolic link
+            try:
+                if not os.path.islink(fp):
+                    total_size_bytes += os.path.getsize(fp)
+                    num_files += 1
+            except Exception as ex:  # Ignore issues from several parallel cleanup processes.
+                pass
+
+    if out == "GB":
+        total_size = total_size_bytes / 1e9
+    elif out == "byte":
+        total_size = total_size_bytes
+    else:
+        raise ValueError("Invalid 'out' unit: '{}', expected one of 'byte' or 'GB'".format(out))
+    return total_size, num_files
+
+
+def setup_storage(storage_dir=__storage__):
+    """
+    Create temporary storage dir if needed, and report on its size.
 
     Returns
     -------
@@ -69,29 +95,17 @@ def setup_storage():
     """
 
     # Create package-wide tmp directory if not already present
-    if not os.path.exists(__storage__):
+    if not os.path.exists(storage_dir):
         try:
-            os.mkdir(__storage__)
+            os.mkdir(storage_dir)
         except Exception as exc:
             err = (
                 "Syncopy core: cannot create temporary storage directory {}. "
                 + "Original error message below\n{}"
             )
-            raise IOError(err.format(__storage__, str(exc)))
-
-    # Check for upper bound of temp directory size
-    with os.scandir(__storage__) as scan:
-        storage_size = 0.0
-        storage_num_files = 0
-        for fle in scan:
-            try:
-                storage_size += fle.stat().st_size / 1024 ** 3
-                storage_num_files += 1
-            # this catches a cleanup by another process
-            except FileNotFoundError:
-                continue
+            raise IOError(err.format(storage_dir, str(exc)))
 
-    return storage_size, storage_num_files
+    return get_dir_size(storage_dir, out="GB")
 
 
 


Test patch:
diff --git a/syncopy/tests/test_datatype_util.py b/syncopy/tests/test_datatype_util.py
new file mode 100644
index 00000000..25823bcf
--- /dev/null
+++ b/syncopy/tests/test_datatype_util.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# Test proper functionality of Syncopy's `ContinuousData` class + subclasses
+#
+
+# Builtin/3rd party package imports
+import os
+import tempfile
+
+# Local imports
+from syncopy.datatype.util import get_dir_size
+
+
+class TestDirSize():
+
+    def test_dirsize(self):
+        with tempfile.TemporaryDirectory() as tdir:
+            fname = "tmpfile"
+            for file_idx in range(20):
+                tf = os.path.join(tdir, fname + str(file_idx))
+                with open(tf, "w") as f:
+                    f.write(f"This is a dummy file {file_idx}.")
+            dir_size_byte, num_files = get_dir_size(tdir, out="byte")
+            assert num_files == 20
+            assert dir_size_byte > 200
+            assert dir_size_byte < 2000
+            assert dir_size_byte == 470
+            dir_size_gb, num_files = get_dir_size(tdir, out="GB")
+            assert dir_size_gb < 1e-6
+
+
+
+
+if __name__ == '__main__':
+
+    T1 = TestDirSize()
+
diff --git a/syncopy/tests/test_statistics.py b/syncopy/tests/test_statistics.py
index e25256c4..f2f5a134 100644
--- a/syncopy/tests/test_statistics.py
+++ b/syncopy/tests/test_statistics.py
@@ -230,7 +230,8 @@ class TestSumStatistics:
         adata = sd.white_noise(100,
                                nSamples=1000,
                                nChannels=2,
-                               samplerate=500)
+                               samplerate=500,
+                               seed=42)
 
         # add simple 60Hz armonic
         adata += sd.harmonic(100,