Repository: SCons/scons

Issue:
Python scanner does not support scanning dynamically generated nodes
**Describe the bug**

The Python scanner from the 'Python' tool is not aware of dynamically generated files because it does not use SCons.Node.FS.find_file. Instead, it looks on disk for the dependency at scanning time.

**Required information**
* Link to SCons Users thread discussing your issue: on today's discord - https://discord.com/channels/571796279483564041/612459756246007829/776833745495261214
* Version of SCons: 4.0.1
* Version of Python: 3.8.6
* Which python distribution if applicable (python.org, cygwin, anaconda, macports, brew,etc)
* How you installed SCons: forked and cloned
* What Platform are you on? (Linux/Windows and which version): Windows 10 x64
* How to reproduce your issue?  Please include a small self contained reproducer. Likely a SConstruct should do for most issues.

I'm still working on a reproducer. Current draft is here: https://github.com/grossag/scons/blob/topic/grossag/pythondynamicscan/test/Scanner/Python.py with dir fixture here: https://github.com/grossag/scons/tree/topic/grossag/pythondynamicscan/test/Scanner/Python but it runs into some other unrelated problems.

General idea is:
1. SConstruct copies a package in.
2. SConstruct runs a script that imports the package at its destination name.
3. SCons should understand that the script depends on the copy action.

* How you invoke scons (The command line you're using "scons --flags some_arguments")

scons --debug=stacktrace .


Discussion:

Resolving patch:
diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py
--- a/SCons/Node/FS.py
+++ b/SCons/Node/FS.py
@@ -3717,7 +3717,10 @@ def filedir_lookup(self, p, fd=None):
         return None
 
     def _find_file_key(self, filename, paths, verbose=None):
-        return (filename, paths)
+        # Note: paths could be a list, which is not hashable. If it is, convert
+        # it to a tuple, which is hashable.
+        paths_entry = tuple(paths) if isinstance(paths, list) else paths
+        return (filename, paths_entry)
 
     @SCons.Memoize.CountDictCall(_find_file_key)
     def find_file(self, filename, paths, verbose=None):
diff --git a/SCons/Scanner/Python.py b/SCons/Scanner/Python.py
--- a/SCons/Scanner/Python.py
+++ b/SCons/Scanner/Python.py
@@ -82,6 +82,40 @@ def find_include_names(node):
     return all_matches
 
 
+def find_import(import_path, search_paths):
+    """
+    Finds the specified import in the various search paths.
+    For an import of "p", it could either result in a file named p.py or
+    p/__init__.py. We can't do two consecutive searches for p then p.py
+    because the first search could return a result that is lower in the
+    search_paths precedence order. As a result, it is safest to iterate over
+    search_paths and check whether p or p.py exists in each path. This allows
+    us to cleanly respect the precedence order.
+
+    If the import is found, returns a tuple containing:
+        1. Discovered dependency node (e.g. p/__init__.py or p.py)
+        2. True if the import was a package, False if the import was a module.
+        3. The Dir node in search_paths that the import is relative to.
+    If the import is not found, returns a tuple containing (None, False, None).
+    Callers should check for failure by checking whether the first entry in the
+    tuple is not None.
+    """
+    for search_path in search_paths:
+        paths = [search_path]
+        # Note: if the same import is present as a package and a module, Python
+        # prefers the package. As a result, we always look for x/__init__.py
+        # before looking for x.py.
+        node = SCons.Node.FS.find_file(import_path + '/__init__.py', paths)
+        if node:
+            return node, True, search_path
+        else:
+            node = SCons.Node.FS.find_file(import_path + '.py', paths)
+            if node:
+                return node, False, search_path
+
+    return None, False, None
+
+
 def scan(node, env, path=()):
     # cache the includes list in node so we only scan it once:
     if node.includes is not None:
@@ -92,10 +126,10 @@ def scan(node, env, path=()):
         # if the same header is included many times.
         node.includes = list(map(SCons.Util.silent_intern, includes))
 
-    # XXX TODO: Sort?
     nodes = []
     if callable(path):
         path = path()
+
     for module, imports in includes:
         is_relative = module.startswith('.')
         if is_relative:
@@ -108,48 +142,57 @@ def scan(node, env, path=()):
             for i in itertools.repeat(None, num_parents):
                 current_dir = current_dir.up()
 
-            search_paths = [current_dir.abspath]
+            search_paths = [current_dir]
             search_string = module_lstripped
         else:
-            search_paths = path
+            search_paths = [env.Dir(p) for p in path]
             search_string = module
 
-        module_components = search_string.split('.')
-        for search_path in search_paths:
-            candidate_path = os.path.join(search_path, *module_components)
-            # The import stored in "module" could refer to a directory or file.
-            import_dirs = []
-            if os.path.isdir(candidate_path):
-                import_dirs = module_components
-
-                # Because this resolved to a directory, there is a chance that
-                # additional imports (e.g. from module import A, B) could refer
-                # to files to import.
-                if imports:
-                    for imp in imports:
-                        file = os.path.join(candidate_path, imp + '.py')
-                        if os.path.isfile(file):
-                            nodes.append(file)
-            elif os.path.isfile(candidate_path + '.py'):
-                nodes.append(candidate_path + '.py')
-                import_dirs = module_components[:-1]
-
-                # We can ignore imports because this resolved to a file. Any
-                # additional imports (e.g. from module.file import A, B) would
-                # only refer to functions in this file.
-
-            # Take a dependency on all __init__.py files from all imported
-            # packages unless it's a relative import. If it's a relative
-            # import, we don't need to take the dependency because Python
-            # requires that all referenced packages have already been imported,
-            # which means that the dependency has already been established.
-            if import_dirs and not is_relative:
-                for i in range(len(import_dirs)):
-                    init_components = module_components[:i+1] + ['__init__.py']
-                    init_path = os.path.join(search_path, *(init_components))
-                    if os.path.isfile(init_path):
-                        nodes.append(init_path)
-                break
+        # If there are no paths, there is no point in parsing includes for this
+        # iteration of the loop.
+        if not search_paths:
+            continue
+
+        module_components = [x for x in search_string.split('.') if x]
+        package_dir = None
+        hit_dir = None
+        if not module_components:
+            # This is just a "from . import x".
+            package_dir = search_paths[0]
+        else:
+            # Translate something like "import x.y" to a call to find_import
+            # with 'x/y' as the path. find_import will then determine whether
+            # we can find 'x/y/__init__.py' or 'x/y.py'.
+            import_node, is_dir, hit_dir = find_import(
+                '/'.join(module_components), search_paths)
+            if import_node:
+                nodes.append(import_node)
+                if is_dir:
+                    package_dir = import_node.dir
+
+        # If the statement was something like "from x import y, z", whether we
+        # iterate over imports depends on whether x was a package or module.
+        # If it was a module, y and z are just functions so we don't need to
+        # search for them. If it was a package, y and z are either packages or
+        # modules and we do need to search for them.
+        if package_dir and imports:
+            for i in imports:
+                import_node, _, _ = find_import(i, [package_dir])
+                if import_node:
+                    nodes.append(import_node)
+
+        # Take a dependency on all __init__.py files from all imported
+        # packages unless it's a relative import. If it's a relative
+        # import, we don't need to take the dependency because Python
+        # requires that all referenced packages have already been imported,
+        # which means that the dependency has already been established.
+        if hit_dir and not is_relative:
+            import_dirs = module_components
+            for i in range(len(import_dirs)):
+                init_path = '/'.join(import_dirs[:i+1] + ['__init__.py'])
+                init_node = SCons.Node.FS.find_file(init_path, [hit_dir])
+                if init_node and init_node not in nodes:
+                    nodes.append(init_node)
 
     return sorted(nodes)
 
diff --git a/SCons/Scanner/PythonTests.py b/SCons/Scanner/PythonTests.py
--- a/SCons/Scanner/PythonTests.py
+++ b/SCons/Scanner/PythonTests.py
@@ -21,6 +21,21 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""
+Unit tests for the Python scanner. These tests validate proper working of the
+Python scanner by confirming that the results of the scan match expectations.
+
+The absolute path tests have strongly-defined behavior in that there is no real
+ambiguity to what they should result in. For example, if you import package x,
+you expect to get x/__init__.py as a dependency.
+
+The relative path tests that reach into ancestor directories do have some
+ambiguity in whether to depend upon __init__.py in those referenced ancestor
+directories. Python only allows these kinds of relative imports if the file is
+part of a package, in which case those ancestor directories' __init__.py files
+have already been imported.
+"""
+
 import SCons.compat
 
 import collections
@@ -226,9 +241,6 @@ def runTest(self):
             'nested1/nested2/nested3/imports_grandparent_module.py')
         path = s.path(env, source=[node])
         deps = s(node, env, path)
-        # Note: there is some ambiguity here in what the scanner should return.
-        # Relative imports require that the referenced packages have already
-        # been imported.
         files = ['nested1/module.py']
         deps_match(self, deps, files)
 
@@ -253,7 +265,59 @@ def runTest(self):
             'nested1/nested2/nested3/imports_parent_then_submodule.py')
         path = s.path(env, source=[node])
         deps = s(node, env, path)
-        files = ['nested1/nested2a/module.py']
+        files = ['nested1/nested2a/__init__.py', 'nested1/nested2a/module.py']
+        deps_match(self, deps, files)
+
+
+class PythonScannerTestImportsModuleWithFunc(unittest.TestCase):
+    def runTest(self):
+        """
+        This test case tests the following import statement:
+        `from simple_package.module1 import somefunc` with somefunc.py existing
+        in the same folder as module1.py. It validates that the scanner doesn't
+        accidentally take a dependency somefunc.py.
+        """
+        env = DummyEnvironment()
+        s = SCons.Scanner.Python.PythonScanner
+        env['ENV']['PYTHONPATH'] = test.workpath('')
+        deps = s(env.File('from_import_simple_package_module1_func.py'), env,
+                 lambda : s.path(env))
+        files = ['simple_package/__init__.py', 'simple_package/module1.py']
+        deps_match(self, deps, files)
+
+
+class PythonScannerTestFromNested1ImportNested2(unittest.TestCase):
+    def runTest(self):
+        """
+        This test case tests the following import statement:
+        `from nested1 import module, nested2`. In this test, module is a Python
+        module and nested2 is a package. Validates that the scanner can handle
+        such mixed imports.
+        """
+        env = DummyEnvironment()
+        s = SCons.Scanner.Python.PythonScanner
+        env['ENV']['PYTHONPATH'] = test.workpath('')
+        deps = s(env.File('from_nested1_import_multiple.py'), env,
+                 lambda : s.path(env))
+        files = ['nested1/__init__.py', 'nested1/module.py',
+                 'nested1/nested2/__init__.py']
+        deps_match(self, deps, files)
+
+
+class PythonScannerTestImportUnknownFiles(unittest.TestCase):
+    def runTest(self):
+        """
+        This test case tests importing files that are not found. If Python
+        really can't find those files, it will fail. But this is intended to
+        test the various failure paths in the scanner to make sure that they
+        don't raise exceptions.
+        """
+        env = DummyEnvironment()
+        s = SCons.Scanner.Python.PythonScanner
+        env['ENV']['PYTHONPATH'] = test.workpath('')
+        deps = s(env.File('imports_unknown_files.py'), env,
+                 lambda : s.path(env))
+        files = []
         deps_match(self, deps, files)
 
 


Test patch:
diff --git a/test/Scanner/Python.py b/test/Scanner/Python.py
new file mode 100644
--- /dev/null
+++ b/test/Scanner/Python.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+#
+# MIT License
+#
+# Copyright The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Functional test for the Python scanner. Validates that the scanner is able to
+find dynamically generated dependencies. The SConstruct copies in the
+dependencies and runs a script. The expected behavior is that the scanner
+picks up these dependencies, so SCons understands that the script shouldn't
+be run until the files are copied.
+"""
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+test.dir_fixture('Python')
+test.run(arguments = '.')
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/Scanner/Python/SConstruct b/test/Scanner/Python/SConstruct
new file mode 100644
--- /dev/null
+++ b/test/Scanner/Python/SConstruct
@@ -0,0 +1,16 @@
+import sys
+
+env = Environment(tools=['python'])
+
+# Copy each file individually instead of copying the dir. This has the benefit
+# of establishing nodes in-memory for each of the resulting files, which helps
+# the scanner work correctly.
+srcDir = env.Dir('to_be_copied')
+for srcNode in srcDir.glob('*'):
+    for destDir in [env.Dir('package1'), env.Dir('package2')]:
+        env.Command(destDir.File(srcNode.name), srcNode,
+                    Copy('$TARGET', '$SOURCE'))
+
+# Don't set a dependency on the copy actions on purpose. Scanner should find
+# the dependencies automatically.
+env.Command('a.out', 'script.py', '$PYTHON $SOURCE $TARGET', PYTHON=sys.executable)
\ No newline at end of file
diff --git a/test/Scanner/Python/sconstest.skip b/test/Scanner/Python/sconstest.skip
new file mode 100644
diff --git a/test/Scanner/Python/script.py b/test/Scanner/Python/script.py
new file mode 100644
--- /dev/null
+++ b/test/Scanner/Python/script.py
@@ -0,0 +1,6 @@
+import package1  # noqa: F401
+import package2  # noqa: F401
+import sys
+
+with open(sys.argv[1], 'w') as f:
+    f.write('test')
diff --git a/test/Scanner/Python/to_be_copied/__init__.py b/test/Scanner/Python/to_be_copied/__init__.py
new file mode 100644
--- /dev/null
+++ b/test/Scanner/Python/to_be_copied/__init__.py
@@ -0,0 +1 @@
+from . import helper  # noqa: F401
\ No newline at end of file
diff --git a/test/Scanner/Python/to_be_copied/helper.py b/test/Scanner/Python/to_be_copied/helper.py
new file mode 100644
diff --git a/test/Scanner/Python/to_be_copied/sconstest.skip b/test/Scanner/Python/to_be_copied/sconstest.skip
new file mode 100644
diff --git a/test/fixture/python_scanner/from_import_simple_package_module1_func.py b/test/fixture/python_scanner/from_import_simple_package_module1_func.py
new file mode 100644
--- /dev/null
+++ b/test/fixture/python_scanner/from_import_simple_package_module1_func.py
@@ -0,0 +1 @@
+from simple_package.module1 import somefunc  # noqa: F401
\ No newline at end of file
diff --git a/test/fixture/python_scanner/from_nested1_import_multiple.py b/test/fixture/python_scanner/from_nested1_import_multiple.py
new file mode 100644
--- /dev/null
+++ b/test/fixture/python_scanner/from_nested1_import_multiple.py
@@ -0,0 +1 @@
+from nested1 import module, nested2  # noqa: F401
\ No newline at end of file
diff --git a/test/fixture/python_scanner/imports_unknown_files.py b/test/fixture/python_scanner/imports_unknown_files.py
new file mode 100644
--- /dev/null
+++ b/test/fixture/python_scanner/imports_unknown_files.py
@@ -0,0 +1,3 @@
+import doesntexist  # noqa: F401
+import notthere.something  # noqa: F401
+from notthere import a, few, things  # noqa: F401
\ No newline at end of file
diff --git a/test/fixture/python_scanner/simple_package/module1.py b/test/fixture/python_scanner/simple_package/module1.py
--- a/test/fixture/python_scanner/simple_package/module1.py
+++ b/test/fixture/python_scanner/simple_package/module1.py
@@ -0,0 +1,2 @@
+def somefunc():
+    return
\ No newline at end of file
diff --git a/test/fixture/python_scanner/simple_package/somefunc.py b/test/fixture/python_scanner/simple_package/somefunc.py
new file mode 100644