Repository: terrapower/armi
Pull request #1984 (2024-10-28 22:53:47)

Update `copyOrWarn` and `getFileSHA1Hash` to account for directories
## What is the change?

`copyOrWarn` can now handle directories. This also includes an edit to `getFileSHA1Hash` so that it can handle directories too.

## Why is the change being made?

Sometimes a directory needs to be copied too. Let's give the directories the attention they deserve!

This supports a case setting downstream that is a directory, not a file path.

---

## Checklist

<!--
    You (the pull requester) should put an `x` in the boxes below you have completed.

    (If a checkbox doesn't apply to your PR, check it anyway.)
-->

- [x] This PR has only [one purpose or idea](https://terrapower.github.io/armi/developer/tooling.html#one-idea-one-pr).
- [x] [Tests](https://terrapower.github.io/armi/developer/tooling.html#test-it) have been added/updated to verify any new/changed code.

<!-- Check the code quality -->

- [x] The code style follows [good practices](https://terrapower.github.io/armi/developer/standards_and_practices.html).
- [x] The commit message(s) follow [good practices](https://terrapower.github.io/armi/developer/tooling.html).

<!-- Check the project-level cruft -->

- [x] The [release notes](https://terrapower.github.io/armi/developer/tooling.html#add-release-notes) have been updated if necessary.
- [x] The [documentation](https://terrapower.github.io/armi/developer/tooling.html#document-it) is still up-to-date in the `doc` folder.
- [x] The dependencies are still up-to-date in `pyproject.toml`.

john-science: Hey Arrielle! Can you re-run the linters?

Thanks!
opotowsky: (working on some tests now + fixing some code, so this isn't quite ready yet)

Patch:
diff --git a/armi/cases/case.py b/armi/cases/case.py
index cadcbaac..faebff6d 100644
--- a/armi/cases/case.py
+++ b/armi/cases/case.py
@@ -855,7 +855,7 @@ def _copyInputsHelper(
     -------
     destFilePath (or origFile) : str
     """
-    sourceName = os.path.basename(sourcePath)
+    sourceName = pathlib.Path(sourcePath).name
     destFilePath = os.path.join(destPath, sourceName)
     try:
         pathTools.copyOrWarn(fileDescription, sourcePath, destFilePath)
@@ -880,9 +880,10 @@ def copyInterfaceInputs(
     This function should now be able to handle the updating of:
 
       - a single file (relative or absolute)
-      - a list of files (relative or absolute), and
+      - a list of files (relative or absolute)
       - a file entry that has a wildcard processing into multiple files.
         Glob is used to offer support for wildcards.
+      - a directory and its contents
 
     If the file paths are absolute, do nothing. The case will be able to find the file.
 
@@ -950,7 +951,7 @@ def copyInterfaceInputs(
                 path = pathlib.Path(f)
                 if not WILDCARD and not RELATIVE:
                     try:
-                        if path.is_absolute() and path.exists() and path.is_file():
+                        if path.is_absolute() and path.exists():
                             # Path is absolute, no settings modification or filecopy needed
                             newFiles.append(path)
                             continue
diff --git a/armi/utils/__init__.py b/armi/utils/__init__.py
index f5943d02..ae9c6844 100644
--- a/armi/utils/__init__.py
+++ b/armi/utils/__init__.py
@@ -24,7 +24,6 @@ import re
 import shutil
 import sys
 import tempfile
-import threading
 import time
 
 from armi import runLog
@@ -38,37 +37,33 @@ _HASH_BUFFER_SIZE = 1024 * 1024
 
 def getFileSHA1Hash(filePath, digits=40):
     """
-    Generate a SHA-1 hash of the input file.
+    Generate a SHA-1 hash of input files.
 
     Parameters
     ----------
     filePath : str
-        Path to file to obtain the SHA-1 hash
+        Path to file or directory to obtain the SHA-1 hash
     digits : int, optional
         Number of digits to include in the hash (40 digit maximum for SHA-1)
     """
     sha1 = hashlib.sha1()
-    with open(filePath, "rb") as f:
-        while True:
-            data = f.read(_HASH_BUFFER_SIZE)
-            if not data:
-                break
-            sha1.update(data)
-
-    return sha1.hexdigest()[:digits]
+    filesToHash = []
+    if os.path.isdir(filePath):
+        for root, _, files in os.walk(filePath):
+            for file in sorted(files):
+                filesToHash.append(os.path.join(root, file))
+    else:
+        filesToHash.append(filePath)
 
+    for file in filesToHash:
+        with open(file, "rb") as f:
+            while True:
+                data = f.read(_HASH_BUFFER_SIZE)
+                if not data:
+                    break
+                sha1.update(data)
 
-def copyWithoutBlocking(src, dest):
-    """
-    Copy a file in a separate thread to avoid blocking while IO completes.
-
-    Useful for copying large files while ARMI moves along.
-    """
-    files = "{} to {}".format(src, dest)
-    runLog.extra("Copying (without blocking) {}".format(files))
-    t = threading.Thread(target=shutil.copy, args=(src, dest))
-    t.start()
-    return t
+    return sha1.hexdigest()[:digits]
 
 
 def getPowerFractions(cs):
diff --git a/armi/utils/pathTools.py b/armi/utils/pathTools.py
index 5eb9d97a..d80354a0 100644
--- a/armi/utils/pathTools.py
+++ b/armi/utils/pathTools.py
@@ -41,29 +41,34 @@ def armiAbsPath(*pathParts):
     return os.path.abspath(os.path.join(*pathParts))
 
 
-def copyOrWarn(fileDescription, sourcePath, destinationPath):
-    """Copy a file, or warn if the file doesn't exist.
+def copyOrWarn(filepathDescription, sourcePath, destinationPath):
+    """Copy a file or directory, or warn if the filepath doesn't exist.
 
     Parameters
     ----------
-    fileDescription : str
+    filepathDescription : str
         a description of the file and/or operation being performed.
     sourcePath : str
-        Path of the file to be copied.
+        Filepath to be copied.
     destinationPath : str
-        Path for the copied file.
+        Copied filepath.
     """
     try:
-        shutil.copy(sourcePath, destinationPath)
+        if os.path.isdir(sourcePath):
+            shutil.copytree(sourcePath, destinationPath, dirs_exist_ok=True)
+        else:
+            shutil.copy(sourcePath, destinationPath)
         runLog.debug(
-            "Copied {}: {} -> {}".format(fileDescription, sourcePath, destinationPath)
+            "Copied {}: {} -> {}".format(
+                filepathDescription, sourcePath, destinationPath
+            )
         )
     except shutil.SameFileError:
         pass
     except Exception as e:
         runLog.warning(
             "Could not copy {} from {} to {}\nError was: {}".format(
-                fileDescription, sourcePath, destinationPath, e
+                filepathDescription, sourcePath, destinationPath, e
             )
         )
 
diff --git a/armi/utils/reportPlotting.py b/armi/utils/reportPlotting.py
index 28433444..936bbec1 100644
--- a/armi/utils/reportPlotting.py
+++ b/armi/utils/reportPlotting.py
@@ -24,7 +24,6 @@ for instance, plot some sequence of objects in a loop at every time node. If you
 to see your memory usage grow inexplicably, you should question any plots that you are
 generating.
 """
-
 import itertools
 import math
 import os
@@ -111,14 +110,10 @@ def plotReactorPerformance(reactor, dbi, buGroups, extension=None, history=None)
         ymin=1.0,
         extension=extension,
     )
-    buVsTime(reactor.name, scalars, extension=extension)
     xsHistoryVsTime(reactor.name, history, buGroups, extension=extension)
     movesVsCycle(reactor.name, scalars, extension=extension)
 
 
-# --------------------------
-
-
 def valueVsTime(name, x, y, key, yaxis, title, ymin=None, extension=None):
     """
     Plots a value vs. time with a standard graph format.
@@ -213,58 +208,6 @@ def keffVsTime(name, time, keff, keffUnc=None, ymin=None, extension=None):
     report.setData("K-Eff", os.path.abspath(figName), report.KEFF_PLOT)
 
 
-def buVsTime(name, scalars, extension=None):
-    r"""
-    produces a burnup and DPA vs. time plot for this case.
-
-    Will add a second axis containing DPA if the scalar column maxDPA exists.
-
-    Parameters
-    ----------
-    name : str
-        reactor.name
-    scalars : dict
-        Scalar values for this case
-    extension : str, optional
-        The file extension for saving the figure
-    """
-    extension = extension or settings.Settings()["outputFileExtension"]
-
-    plt.figure()
-    try:
-        plt.plot(scalars["time"], scalars["maxBuI"], ".-", label="Driver")
-    except ValueError:
-        runLog.warning(
-            "Incompatible axis length in burnup plot. Time has {0}, bu has {1}. Skipping"
-            "".format(len(scalars["time"]), len(scalars["maxBuI"]))
-        )
-        plt.close(1)
-        return
-
-    plt.plot(scalars["time"], scalars["maxBuF"], ".-", label="Feed")
-    plt.xlabel("Time (yr)")
-    plt.ylabel("BU (%FIMA)")
-    plt.grid(color="0.70")
-    plt.legend(loc="lower left")
-    title = "Maximum burnup"
-    if scalars["maxDPA"]:
-        plt.twinx()
-        plt.plot(scalars["time"], scalars["maxDPA"], "r--", label="dpa")
-        plt.legend(loc="lower right")
-        plt.ylabel("dpa")
-        title += " and DPA"
-
-    title += " for " + name
-
-    plt.title(title)
-    plt.legend(loc="lower right")
-    figName = name + ".bu." + extension
-    plt.savefig(figName)
-    plt.close(1)
-
-    report.setData("Burnup Plot", os.path.abspath(figName), report.BURNUP_PLOT)
-
-
 def xsHistoryVsTime(name, history, buGroups, extension=None):
     r"""
     Plot cross section history vs. time.
@@ -675,7 +618,6 @@ def createPlotMetaData(
     -------
     metadata : dict
         Dictionary with all plot metadata information
-
     """
     metadata = {}
 
diff --git a/doc/release/0.4.rst b/doc/release/0.4.rst
index 84bd6f38..fd9ed554 100644
--- a/doc/release/0.4.rst
+++ b/doc/release/0.4.rst
@@ -26,6 +26,7 @@ New Features
 #. Improve performance by changing the lattice physics interface so that cross sections are not updated on ``everyNode`` calls during coupled calculations (`PR#1963 <https://github.com/terrapower/armi/pull/1963>`_)
 #. Improve efficiency of reaction rate calculations. (`PR#1887 <https://github.com/terrapower/armi/pull/1887>`_)
 #. Adding new options for simplifying 1D cross section modeling. (`PR#1949 <https://github.com/terrapower/armi/pull/1949>`_)
+#. Updating ``copyOrWarn`` and ``getFileSHA1Hash`` to support directories. (`PR#1984 <https://github.com/terrapower/armi/pull/1984>`_)
 #. TBD
 
 API Changes
@@ -38,6 +39,7 @@ API Changes
 #. Renaming ``Reactor.moveList`` to ``Reactor.moves``. (`PR#1881 <https://github.com/terrapower/armi/pull/1881>`_)
 #. ``copyInterfaceInputs`` no longer requires a valid setting object. (`PR#1934 <https://github.com/terrapower/armi/pull/1934>`_)
 #. Removing ``buildEqRingSchedule``. (`PR#1928 <https://github.com/terrapower/armi/pull/1928>`_)
+#. Removing broken plot ``buVsTime``. (`PR#1994 <https://github.com/terrapower/armi/pull/1994>`_)
 #. Allowing for unknown Flags when opening a DB. (`PR#1844 <https://github.com/terrapower/armi/pull/1835>`_)
 #. Removing ``Assembly.doubleResolution()``. (`PR#1951 <https://github.com/terrapower/armi/pull/1951>`_)
 #. Removing ``assemblyLists.py`` and the ``AssemblyList`` class. (`PR#1891 <https://github.com/terrapower/armi/pull/1891>`_)


Test patch:
diff --git a/armi/utils/tests/test_pathTools.py b/armi/utils/tests/test_pathTools.py
index 47c4a8b8..2b6f046d 100644
--- a/armi/utils/tests/test_pathTools.py
+++ b/armi/utils/tests/test_pathTools.py
@@ -26,6 +26,41 @@ THIS_DIR = os.path.dirname(__file__)
 
 
 class PathToolsTests(unittest.TestCase):
+    def test_copyOrWarnFile(self):
+        with TemporaryDirectoryChanger():
+            # Test a successful copy
+            path = "test.txt"
+            pathCopy = "testcopy.txt"
+            with open(path, "w") as f1:
+                f1.write("test")
+            pathTools.copyOrWarn("Test File", path, pathCopy)
+            self.assertTrue(os.path.exists(pathCopy))
+
+            # Test an exception
+            with self.assertRaises(Exception) as e:
+                pathTools.copyOrWarn("Test File", "FileDoesntExist.txt", pathCopy)
+                self.assertIn("Could not copy", e)
+                self.assertIn("No such file or directory", e)
+
+    def test_copyOrWarnDir(self):
+        with TemporaryDirectoryChanger():
+            # Test a successful copy
+            pathDir = "testDir"
+            path = os.path.join(pathDir, "test.txt")
+            pathDirCopy = "testcopy"
+            os.mkdir(pathDir)
+            with open(path, "w") as f1:
+                f1.write("test")
+            pathTools.copyOrWarn("Test File", pathDir, pathDirCopy)
+            self.assertTrue(os.path.exists(pathDirCopy))
+            self.assertTrue(os.path.exists(os.path.join(pathDirCopy, "test.txt")))
+
+            # Test an exception
+            with self.assertRaises(Exception) as e:
+                pathTools.copyOrWarn("Test File", "DirDoesntExist", pathDirCopy)
+                self.assertIn("Could not copy", e)
+                self.assertIn("No such file or directory", e)
+
     def test_separateModuleAndAttribute(self):
         self.assertRaises(
             ValueError, pathTools.separateModuleAndAttribute, r"path/with/no/colon"
diff --git a/armi/utils/tests/test_reportPlotting.py b/armi/utils/tests/test_reportPlotting.py
index ae2cd6f2..5e0268bf 100644
--- a/armi/utils/tests/test_reportPlotting.py
+++ b/armi/utils/tests/test_reportPlotting.py
@@ -22,7 +22,6 @@ from armi.reactor.tests import test_reactors
 from armi.tests import TEST_ROOT
 from armi.utils.directoryChangers import TemporaryDirectoryChanger
 from armi.utils.reportPlotting import (
-    buVsTime,
     createPlotMetaData,
     keffVsTime,
     movesVsCycle,
@@ -98,19 +97,6 @@ class TestRadar(unittest.TestCase):
         self.assertTrue(os.path.exists("R-armiRunSmallest.val.png"))
         self.assertGreater(os.path.getsize("R-armiRunSmallest.val.png"), 0)
 
-    def test_buVsTime(self):
-        name = "buvstime"
-        scalars = {
-            "time": [1, 2, 3, 4],
-            "maxBuI": [6, 7, 8, 9],
-            "maxBuF": [6, 7, 8, 9],
-            "maxDPA": [6, 7, 8, 9],
-        }
-        figName = name + ".bu.png"
-        buVsTime(name, scalars, "png")
-        self.assertTrue(os.path.exists(figName))
-        self.assertGreater(os.path.getsize(figName), 0)
-
     def test_movesVsCycle(self):
         name = "movesVsCycle"
         scalars = {
diff --git a/armi/utils/tests/test_utils.py b/armi/utils/tests/test_utils.py
index 826a155e..f0e0b650 100644
--- a/armi/utils/tests/test_utils.py
+++ b/armi/utils/tests/test_utils.py
@@ -24,26 +24,47 @@ from armi.reactor.tests.test_reactors import loadTestReactor
 from armi.settings.caseSettings import Settings
 from armi.tests import mockRunLogs
 from armi.utils import (
+    codeTiming,
     directoryChangers,
-    getPowerFractions,
-    getCycleNames,
     getAvailabilityFactors,
-    getStepLengths,
-    getCycleLengths,
     getBurnSteps,
+    getCumulativeNodeNum,
+    getCycleLengths,
+    getCycleNames,
+    getCycleNodeFromCumulativeNode,
+    getCycleNodeFromCumulativeStep,
+    getFileSHA1Hash,
     getMaxBurnSteps,
     getNodesPerCycle,
-    getCycleNodeFromCumulativeStep,
-    getCycleNodeFromCumulativeNode,
+    getPowerFractions,
     getPreviousTimeNode,
-    getCumulativeNodeNum,
+    getStepLengths,
     hasBurnup,
-    codeTiming,
     safeCopy,
 )
 
 
 class TestGeneralUtils(unittest.TestCase):
+    def test_getFileSHA1Hash(self):
+        with directoryChangers.TemporaryDirectoryChanger():
+            path = "test.txt"
+            with open(path, "w") as f1:
+                f1.write("test")
+            sha = getFileSHA1Hash(path)
+            self.assertIn("a94a8", sha)
+
+    def test_getFileSHA1HashDir(self):
+        with directoryChangers.TemporaryDirectoryChanger():
+            pathDir = "testDir"
+            path1 = os.path.join(pathDir, "test1.txt")
+            path2 = os.path.join(pathDir, "test2.txt")
+            os.mkdir(pathDir)
+            for i, path in enumerate([path1, path2]):
+                with open(path, "w") as f1:
+                    f1.write(f"test{i}")
+            sha = getFileSHA1Hash(pathDir)
+            self.assertIn("ccd13", sha)
+
     def test_mergeableDictionary(self):
         mergeableDict = utils.MergeableDict()
         normalDict = {"luna": "thehusky", "isbegging": "fortreats", "right": "now"}