Repository: ESMCI/cime

Issue:
imp module is deprecated; replace with importlib
I notice that we have a few uses of the 'imp' module in CIME. According to python documentation, this was deprecated as of python3.4, and we should switch to using importlib.

That said, I'm adding another use of the 'imp' module that follows our current patterns, because it seems like it will be easiest to stay consistent and then switch everything at once. The right way to switch our usage of imp.load_source over to importlib seems to depend on the python versions we want to support. According to https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path, for python3.5+, we'll need this lovely bit of code (apparently the python maintainers really don't want you to do this):

```python
import importlib.util
spec = importlib.util.spec_from_file_location("module.name", "/path/to/file.py")
foo = importlib.util.module_from_spec(spec)
spec.loader.exec_module(foo)
```

See also https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly, which adds a line, `sys.modules[module_name] = module`; I'm not sure if that's needed.

So I guess, if we are comfortable only supporting python3.5+, we should add a utility function that does the above and then switch our current usage to use that.

Thoughts?



Discussion:
I agree.
I thought that we had already made a decision to support only 3.7 and newer.  https://github.com/ESMCI/cime/blob/master/.github/workflows/srt.yml#L21

Resolving patch:
diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py
--- a/scripts/lib/CIME/build.py
+++ b/scripts/lib/CIME/build.py
@@ -1,9 +1,12 @@
 """
 functions for building CIME models
 """
-import glob, shutil, time, threading, subprocess, imp
+import glob, shutil, time, threading, subprocess
 from CIME.XML.standard_module_setup  import *
-from CIME.utils                 import get_model, analyze_build_log, stringify_bool, run_and_log_case_status, get_timestamp, run_sub_or_cmd, run_cmd, get_batch_script_for_job, gzip_existing_file, safe_copy, check_for_python, get_logging_options
+from CIME.utils                 import get_model, analyze_build_log, \
+    stringify_bool, run_and_log_case_status, get_timestamp, run_sub_or_cmd, \
+    run_cmd, get_batch_script_for_job, gzip_existing_file, safe_copy, \
+    check_for_python, get_logging_options, import_from_file
 from CIME.provenance            import save_build_provenance as save_build_provenance_sub
 from CIME.locked_files          import lock_file, unlock_file
 from CIME.XML.files             import Files
@@ -543,7 +546,8 @@ def _create_build_metadata_for_component(config_dir, libroot, bldroot, case):
     """
     bc_path = os.path.join(config_dir, "buildlib_cmake")
     expect(os.path.exists(bc_path), "Missing: {}".format(bc_path))
-    buildlib = imp.load_source("buildlib_cmake", os.path.join(config_dir, "buildlib_cmake"))
+    buildlib = import_from_file("buildlib_cmake", os.path.join(config_dir,
+                                                               "buildlib_cmake"))
     cmake_args = buildlib.buildlib(bldroot, libroot, case)
     return "" if cmake_args is None else cmake_args
 
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -2,7 +2,9 @@
 Common functions used by cime python scripts
 Warning: you cannot use CIME Classes in this module as it causes circular dependencies
 """
-import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, fnmatch
+import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, \
+    importlib, fnmatch
+import importlib.util
 import errno, signal, warnings, filecmp
 import stat as statlib
 import six
@@ -15,6 +17,19 @@
 TESTS_FAILED_ERR_CODE = 100
 logger = logging.getLogger(__name__)
 
+def import_from_file(name, file_path):
+    loader = importlib.machinery.SourceFileLoader(name, file_path)
+
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+
+    module = importlib.util.module_from_spec(spec)
+
+    sys.modules[name] = module
+
+    spec.loader.exec_module(module)
+
+    return module
+
 @contextmanager
 def redirect_stdout(new_target):
     old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
@@ -414,7 +429,7 @@ def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None,
 
     if not do_run_cmd:
         try:
-            mod = imp.load_source(subname, cmd)
+            mod = import_from_file(subname, cmd)
             logger.info("   Calling {}".format(cmd))
             # Careful: logfile code is not thread safe!
             if logfile:


Test patch:
diff --git a/scripts/lib/CIME/tests/test_utils.py b/scripts/lib/CIME/tests/test_utils.py
--- a/scripts/lib/CIME/tests/test_utils.py
+++ b/scripts/lib/CIME/tests/test_utils.py
@@ -2,10 +2,12 @@
 
 import os
 import sys
+import tempfile
 
 import unittest
 from unittest import mock
-from CIME.utils import indent_string, run_and_log_case_status
+from CIME.utils import indent_string, run_and_log_case_status, \
+    import_from_file
 
 from . import utils
 
@@ -87,6 +89,19 @@ def assertMatchAllLines(self, tempdir, test_lines):
 
         self.assertTrue(result, msg="\n".join(error))
 
+    def test_import_from_file(self):
+        with tempfile.NamedTemporaryFile() as fd:
+            fd.writelines([
+                b"def test():\n",
+                b"  return 'value'",
+            ])
+
+            fd.flush()
+
+            module = import_from_file("test.py", fd.name)
+
+            assert module.test() == "value"
+
     def test_run_and_log_case_status(self):
         test_lines = [
             "00:00:00 default starting \n",
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -22,7 +22,9 @@
 
 import collections
 
-from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit, safe_copy, CIMEError, get_cime_root, get_src_root, Timeout
+from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit, \
+    safe_copy, CIMEError, get_cime_root, get_src_root, Timeout, \
+    import_from_file
 import get_tests
 import CIME.test_scheduler, CIME.wait_for_tests
 from  CIME.test_scheduler import TestScheduler
@@ -2431,19 +2433,16 @@ def test_env_loading(self):
     ###########################################################################
     def test_case_submit_interface(self):
     ###########################################################################
-        try:
-            import imp
-        except ImportError:
-            print("imp not found, skipping case.submit interface test")
-            return
         # the current directory may not exist, so make sure we are in a real directory
         os.chdir(os.getenv("HOME"))
         sys.path.append(TOOLS_DIR)
         case_submit_path = os.path.join(TOOLS_DIR, "case.submit")
-        submit_interface = imp.load_source("case_submit_interface", case_submit_path)
+
+        module = import_from_file("case.submit", case_submit_path)
+
         sys.argv = ["case.submit", "--batch-args", "'random_arguments_here.%j'",
                     "--mail-type", "fail", "--mail-user", "'random_arguments_here.%j'"]
-        submit_interface._main_func(None, True)
+        module._main_func(None, True)
 
     ###########################################################################
     def test_xml_caching(self):