Repository: ESMCI/cime

Issue:
Create hash at create_newcase time.
To help uniquely identify cases in experiment data bases, create a hash using:

- the create_newcase command arguments,
- the date and time,
- the local host name
- the user id.

Store the value of this hash in env_case.xml.  Only way to change it is re-create the case.

This should be enough to create unique hashes for every case created with cime.



Discussion:
So, what happens when folks use xmlchange? Does it necessitate creating a new hash? We are presently using LID to distinguish completed jobs but that's runtime.
No.  It just needs to be unique for the invocation of create_newcase and not try to capture anything about the content of the case.  The rest of the provenance can do that.  But this hash will be how we add more provenance from things like zstash.
So, if there are multiple jobs submitted from that case directory, they will share the hash then. Then it's not unique for jobs.
right.  Just for the case.   Its finer-grained then --case-group which multiple cases might have but not for each case.submit invocation in the casedir.

Resolving patch:
diff --git a/scripts/lib/CIME/case/case.py b/scripts/lib/CIME/case/case.py
--- a/scripts/lib/CIME/case/case.py
+++ b/scripts/lib/CIME/case/case.py
@@ -5,7 +5,7 @@
 through the Case module.
 """
 from copy import deepcopy
-import glob, os, shutil, math, six, time
+import glob, os, shutil, math, six, time, hashlib
 from CIME.XML.standard_module_setup import *
 #pylint: disable=import-error,redefined-builtin
 from six.moves import input
@@ -268,6 +268,7 @@ def copy(self, newcasename, newcaseroot, newcimeroot=None, newsrcroot=None):
         newcase.set_value("CASEROOT",newcaseroot)
         newcase.set_value("CONTINUE_RUN","FALSE")
         newcase.set_value("RESUBMIT",0)
+        newcase.set_value("CASE_HASH", newcase.new_hash())
 
         # Important, and subtle: Writability should NOT be copied because
         # this allows the copy to be modified without needing a "with" statement
@@ -1686,6 +1687,7 @@ def create(self, casename, srcroot, compset_name, grid_name,
             self.set_lookup_value("CASE", os.path.basename(casename))
             self.set_lookup_value("CASEROOT", self._caseroot)
             self.set_lookup_value("SRCROOT", srcroot)
+            self.set_lookup_value("CASE_HASH", self.new_hash())
             # if the top level user_mods_dir contains a config_grids.xml file and
             # gridfile was not set on the command line, use it.
             if user_mods_dir:
@@ -1728,6 +1730,18 @@ def create(self, casename, srcroot, compset_name, grid_name,
 
             raise
 
+    def new_hash(self):
+        """ Creates a hash
+        """
+        args = "".join(sys.argv)
+        ctime = time.strftime("%Y-%m-%d %H:%M:%S")
+        hostname = os.environ["HOSTNAME"]
+        user = os.environ["USER"]
+
+        data = "{}{}{}{}".format(args, ctime, hostname, user)
+
+        return hashlib.sha256(data.encode()).hexdigest()
+
     def is_save_timing_dir_project(self,project):
         """
         Check whether the project is permitted to archive performance data in the location


Test patch:
diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py
--- a/scripts/lib/CIME/tests/test_case.py
+++ b/scripts/lib/CIME/tests/test_case.py
@@ -7,6 +7,134 @@
 
 from . import utils
 
+class TestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.srcroot = os.path.abspath(cime_utils.get_cime_root())
+        self.tempdir = utils.TemporaryDirectory()
+
+        self.mock = utils.Mocker()
+        self.mock.patch(Case, "read_xml")
+        Case._force_read_only = False # pylint: disable=protected-access
+        self._time_strftime = self.mock.patch('time.strftime', ret='00:00:00')
+        self.mock.patch("sys.argv",
+                        ret=[
+                            "/src/create_newcase",
+                            "--machine",
+                            "docker"
+                        ], is_property=True)
+
+    def test_new_hash(self):
+        with self.tempdir as tempdir:
+            with Case(tempdir) as case:
+                os.environ["USER"] = "root"
+                os.environ["HOSTNAME"] = "host1"
+
+                expected = "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a"
+
+                # Check idempotency
+                for _ in range(2):
+                    value = case.new_hash()
+
+                    self.assertTrue(value == expected,
+                                    "{} != {}".format(value, expected))
+
+                os.environ["USER"] = "johndoe"
+
+                expected = "bb59f1c473ac07e9dd30bfab153c0530a777f89280b716cf42e6fe2f49811a6e"
+
+                value = case.new_hash()
+
+                self.assertTrue(value == expected,
+                                "{} != {}".format(value, expected))
+
+    def test_copy(self):
+        expected_first_hash = "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a"
+        expected_second_hash = "3561339a49daab999e3c4ea2f03a9c6acc33296a5bc35f1bfb82e7b5e10bdf38"
+
+        with self.tempdir as tempdir:
+            caseroot = os.path.join(tempdir, "test1")
+            with Case(caseroot, read_only=False) as case:
+                os.environ["USER"] = "root"
+                os.environ["HOSTNAME"] = "host1"
+                os.environ["CIME_MODEL"] = "cesm"
+
+                # Need to mock all of these to prevent errors with xml files
+                # just want to ensure `create` calls `set_lookup_value`
+                # correctly.
+                _configure = self.mock.patch(case, "configure")
+                _create_caseroot = self.mock.patch(case, "create_caseroot")
+                _apply_user_mods = self.mock.patch(case, "apply_user_mods")
+                _lock_file = self.mock.patch("CIME.case.case.lock_file")
+                _set_lookup_value = self.mock.patch(case, "set_lookup_value")
+
+                srcroot = os.path.abspath(os.path.join(
+                    os.path.dirname(__file__), "../../../../../"))
+                case.create("test1", srcroot, "A", "f19_g16_rx1",
+                            machine_name="ubuntu-latest")
+
+                # Check that they're all called
+                _configure.assert_called_with(args=["A", "f19_g16_rx1"])
+                _create_caseroot.assert_called()
+                _apply_user_mods.assert_called()
+                _lock_file.assert_called()
+
+                self.assertTrue(_set_lookup_value.calls[-1]['args'][0] ==
+                                "CASE_HASH")
+                self.assertTrue(_set_lookup_value.calls[-1]['args'][1] ==
+                                expected_first_hash)
+
+                _set_value = self.mock.patch(case, "set_value",
+                                             ret=utils.Mocker())
+
+                # simulate change
+                self._time_strftime.ret = "10:00:00"
+                self.mock.patch("sys.argv",
+                                ret=[
+                                    "/src/create_clone"
+                                ], is_property=True, update_value_only=True)
+
+                case.copy("test2", "{}_2".format(tempdir))
+
+                self.assertTrue(_set_value.calls[-1]['args'][0] ==
+                                "CASE_HASH")
+                self.assertTrue(_set_value.calls[-1]['args'][1] ==
+                                expected_second_hash)
+
+
+    def test_create(self):
+        with self.tempdir as tempdir:
+            caseroot = os.path.join(tempdir, "test1")
+            with Case(caseroot, read_only=False) as case:
+                os.environ["USER"] = "root"
+                os.environ["HOSTNAME"] = "host1"
+                os.environ["CIME_MODEL"] = "cesm"
+
+                # Need to mock all of these to prevent errors with xml files
+                # just want to ensure `create` calls `set_lookup_value`
+                # correctly.
+                _configure = self.mock.patch(case, 'configure')
+                _create_caseroot = self.mock.patch(case, 'create_caseroot')
+                _apply_user_mods = self.mock.patch(case, 'apply_user_mods')
+                _lock_file = self.mock.patch('CIME.case.case.lock_file')
+                _set_lookup_value = self.mock.patch(case, "set_lookup_value")
+
+                srcroot = os.path.abspath(os.path.join(
+                    os.path.dirname(__file__), "../../../../../"))
+                case.create("test1", srcroot, "A", "f19_g16_rx1",
+                            machine_name="ubuntu-latest")
+
+                # Check that they're all called
+                _configure.assert_called_with(args=["A", "f19_g16_rx1"])
+                _create_caseroot.assert_called()
+                _apply_user_mods.assert_called()
+                _lock_file.assert_called()
+
+                self.assertTrue(_set_lookup_value.calls[-1]['args'][0] ==
+                                "CASE_HASH")
+                self.assertTrue(_set_lookup_value.calls[-1]['args'][1] ==
+                                "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a")
+
 class TestCase_RecordCmd(unittest.TestCase):
 
     def setUp(self):
@@ -55,7 +183,7 @@ def test_init(self):
             "cd \"${CASEDIR}\"\n\n",
         ]
 
-        calls = open_mock.ret.method_calls["writelines"][0]["args"][0]
+        calls = open_mock.ret.method_calls["writelines"][0]["args"][0] # pylint: disable=no-member
 
         self.assert_calls_match(calls, expected)
 
@@ -86,7 +214,7 @@ def test_sub_relative(self):
             "cd \"${CASEDIR}\"\n\n",
         ]
 
-        calls = open_mock.ret.method_calls["writelines"][0]["args"][0]
+        calls = open_mock.ret.method_calls["writelines"][0]["args"][0] # pylint: disable=no-member
 
         self.assert_calls_match(calls, expected)
 
@@ -110,7 +238,7 @@ def test_cmd_arg(self):
             "/some/custom/command arg1\n\n",
         ]
 
-        calls = open_mock.ret.method_calls["writelines"][0]["args"][0]
+        calls = open_mock.ret.method_calls["writelines"][0]["args"][0] # pylint: disable=no-member
 
         self.assert_calls_match(calls, expected)
 
diff --git a/scripts/lib/CIME/tests/utils.py b/scripts/lib/CIME/tests/utils.py
--- a/scripts/lib/CIME/tests/utils.py
+++ b/scripts/lib/CIME/tests/utils.py
@@ -45,6 +45,31 @@ def method_calls(self):
     def ret(self):
         return self._ret
 
+    @ret.setter
+    def ret(self, value):
+        self._ret = value
+
+    def assert_called(self):
+        assert len(self.calls) > 0
+
+    def assert_called_with(self, i=None, args=None, kwargs=None):
+        if i is None:
+            i = 0
+
+        call = self.calls[i]
+
+        if args is not None:
+            _call_args = set(call['args'])
+            _exp_args = set(args)
+            assert _exp_args <= _call_args, "Got {} missing {}".format(
+                _call_args, _exp_args-_call_args)
+
+        if kwargs is not None:
+            call_kwargs = call['kwargs']
+
+            for x, y in kwargs.items():
+                assert call_kwargs[x] == y, "Missing {}".format(x)
+
     def __getattr__(self, name):
         if name in self._method_calls:
             new_method = self._method_calls[name]
@@ -81,19 +106,22 @@ def revert_mocks(self):
             else:
                 setattr(module, method, m)
 
-    def patch(self, module, method=None, ret=None, is_property=False):
+    def patch(self, module, method=None, ret=None, is_property=False,
+              update_value_only=False):
         rv = None
         if isinstance(module, str):
             x = module.split('.')
             main = '.'.join(x[:-1])
-            self._orig.append((getattr(sys.modules[main], x[-1]), main, x[-1]))
+            if not update_value_only:
+                self._orig.append((getattr(sys.modules[main], x[-1]), main, x[-1]))
             if is_property:
                 setattr(sys.modules[main], x[-1], ret)
             else:
                 rv = Mocker(ret, cmd=x[-1])
                 setattr(sys.modules[main], x[-1], rv)
         elif method != None:
-            self._orig.append((getattr(module, method), module, method))
+            if not update_value_only:
+                self._orig.append((getattr(module, method), module, method))
             rv = Mocker(ret)
             setattr(module, method, rv)
         else: