Repository: SCons/scons

Issue:
Copy(somedir, Glob('otherdir/*')) fails because src list is dropped
This issue was originally created at: 2015-07-27 06:17:29.
This issue was reported by: `knut3`.
knut3 said at 2015-07-27 06:17:29
>Hi,
> 
I want to copy all files from a directory to another in a builder. I use:
 
```py
Execute(Copy('somedir', Glob('otherdir/*')))
```
 
This fails with error: `No such file or directory`
 
`Copy()` is implemented like this:
 
```py
def copy_func(dest, src):
    SCons.Node.FS.invalidate_node_memos(dest)
    if SCons.Util.is_List(src) and os.path.isdir(dest):
        for file in src:
            shutil.copy2(file, dest)
        return 0
    elif os.path.isfile(src):
        return shutil.copy2(src, dest)
    else:
        return shutil.copytree(src, dest, 1)
 
Copy = ActionFactory(copy_func,
                     lambda dest, src: 'Copy("%s", "%s")' % (dest, src),
                     convert=str)
```
 
---
 
I added some prints to see what's happening here:
 
```py
SCons.Util.is_List(src) => false
type(src) => str
```
 
`src` is converted to str by mistake. Thus, the wrong if clause is taken in `copy_func()`. This is caused by `convert=str` of the `ActionFactory`. If I remove this, scons fails somewhere else, but the list is preserved correctly.




Discussion:
This seems a valid complaint, though - the code at the head of the function unconditionally converts the two args, then proceeds to do checks based on the type - these two lines, which don't seem to be in the version the original poster commented on:
```py
    dest = str(dest)
    src = str(src)
...
    if SCons.Util.is_List(src) and os.path.isdir(dest):
```
My previous comment is hard to parse... anyway, just arrived at this issue by other means and did a "git blame" on the Copy code, and we come down to commit 5e3450b055 - which added the unconditional string conversions above as part of adding symlink support. That replaces the earlier `convert=str` mentioned in the original report. As a result it is impossible for the src argument to be a list, which is unusual for SCons syntax in general, but actually matches the extremely limited documentation (there's no docstring, only the manpage entry):

> Returns an Action object that will copy the `src` source file or directory to the `dest` destination file or directory.

So, if taken literally, "file or directory" would suggest you can't pass a list. Do we believe that?
Seems like a very small patch can fix this, if we decide it is *not* working as intended - which I now think it is not.
```py
index 65bd75ac2..274e9eb34 100644
--- a/SCons/Defaults.py
+++ b/SCons/Defaults.py
@@ -251,7 +251,10 @@ def copy_func(dest, src, symlinks=True):
     """
 
     dest = str(dest)
-    src = str(src)
+    if SCons.Util.is_List(src):
+        src = [str(n) for n in src]
+    else:
+        src = str(src)
 
     SCons.Node.FS.invalidate_node_memos(dest)
     if SCons.Util.is_List(src) and os.path.isdir(dest):
```

The reason I think this is just an oversight is just a few lines later there's an `is_List` check, and it does copies in a loop if so. @bdbaddog - should I generate a PR?
Additional note: the original example would be better written as: 
```py
Execute(Copy('somedir', Glob('otherdir/*', strings=True)))
```
because while a conversion from list-of-Nodes can take place inside the function called by the builder, this happens after the string function prints the command - so the output will look garbled (does not affect operation)

Resolving patch:
diff --git a/SCons/Defaults.py b/SCons/Defaults.py
--- a/SCons/Defaults.py
+++ b/SCons/Defaults.py
@@ -41,10 +41,12 @@
 import SCons.Builder
 import SCons.CacheDir
 import SCons.Environment
+import SCons.Errors
 import SCons.PathList
 import SCons.Scanner.Dir
 import SCons.Subst
 import SCons.Tool
+import SCons.Util
 
 # A placeholder for a default Environment (for fetching source files
 # from source code management systems and the like).  This must be
@@ -79,7 +81,6 @@ def DefaultEnvironment(*args, **kw):
     """
     global _default_env
     if not _default_env:
-        import SCons.Util
         _default_env = SCons.Environment.Environment(*args, **kw)
         _default_env.Decider('content')
         global DefaultEnvironment
@@ -157,15 +158,19 @@ def SharedFlagChecker(source, target, env):
 ActionFactory = SCons.Action.ActionFactory
 
 
-def get_paths_str(dest):
-    # If dest is a list, we need to manually call str() on each element
+def get_paths_str(dest) -> str:
+    """Generates a string from *dest* for use in a strfunction.
+
+    If *dest* is a list, manually converts each elem to a string.
+    """
+    def quote(arg):
+        return f'"{arg}"'
+
     if SCons.Util.is_List(dest):
-        elem_strs = []
-        for element in dest:
-            elem_strs.append('"' + str(element) + '"')
-        return '[' + ', '.join(elem_strs) + ']'
+        elem_strs = [quote(d) for d in dest]
+        return f'[{", ".join(elem_strs)}]'
     else:
-        return '"' + str(dest) + '"'
+        return quote(dest)
 
 
 permission_dic = {
@@ -187,8 +192,14 @@ def get_paths_str(dest):
 }
 
 
-def chmod_func(dest, mode):
-    import SCons.Util
+def chmod_func(dest, mode) -> None:
+    """Implementation of the Chmod action function.
+
+    *mode* can be either an integer (normally expressed in octal mode,
+    as in 0o755) or a string following the syntax of the POSIX chmod
+    command (for example "ugo+w"). The latter must be converted, since
+    the underlying Python only takes the numeric form.
+    """
     from string import digits
     SCons.Node.FS.invalidate_node_memos(dest)
     if not SCons.Util.is_List(dest):
@@ -231,55 +242,85 @@ def chmod_func(dest, mode):
                     os.chmod(str(element), curr_perm & ~new_perm)
 
 
-def chmod_strfunc(dest, mode):
-    import SCons.Util
+def chmod_strfunc(dest, mode) -> str:
+    """strfunction for the Chmod action function."""
     if not SCons.Util.is_String(mode):
-        return 'Chmod(%s, 0%o)' % (get_paths_str(dest), mode)
+        return f'Chmod({get_paths_str(dest)}, {mode:#o})'
     else:
-        return 'Chmod(%s, "%s")' % (get_paths_str(dest), str(mode))
+        return f'Chmod({get_paths_str(dest)}, "{mode}")'
+
 
 
 Chmod = ActionFactory(chmod_func, chmod_strfunc)
 
 
-def copy_func(dest, src, symlinks=True):
-    """
-    If symlinks (is true), then a symbolic link will be
+
+def copy_func(dest, src, symlinks=True) -> int:
+    """Implementation of the Copy action function.
+
+    Copies *src* to *dest*.  If *src* is a list, *dest* must be
+    a directory, or not exist (will be created).
+
+    Since Python :mod:`shutil` methods, which know nothing about
+    SCons Nodes, will be called to perform the actual copying,
+    args are converted to strings first.
+
+    If *symlinks* evaluates true, then a symbolic link will be
     shallow copied and recreated as a symbolic link; otherwise, copying
     a symbolic link will be equivalent to copying the symbolic link's
     final target regardless of symbolic link depth.
     """
 
     dest = str(dest)
-    src = str(src)
+    src = [str(n) for n in src] if SCons.Util.is_List(src) else str(src)
 
     SCons.Node.FS.invalidate_node_memos(dest)
-    if SCons.Util.is_List(src) and os.path.isdir(dest):
+    if SCons.Util.is_List(src):
+        # this fails only if dest exists and is not a dir
+        try:
+            os.makedirs(dest, exist_ok=True)
+        except FileExistsError:
+            raise SCons.Errors.BuildError(
+                errstr=(
+                    'Error: Copy() called with a list of sources, '
+                    'which requires target to be a directory, '
+                    f'but "{dest}" is not a directory.'
+                )
+            )
         for file in src:
             shutil.copy2(file, dest)
         return 0
+
     elif os.path.islink(src):
         if symlinks:
-            return os.symlink(os.readlink(src), dest)
-        else:
-            return copy_func(dest, os.path.realpath(src))
+            os.symlink(os.readlink(src), dest)
+            return 0
+
+        return copy_func(dest, os.path.realpath(src))
+
     elif os.path.isfile(src):
         shutil.copy2(src, dest)
         return 0
+
     else:
         shutil.copytree(src, dest, symlinks)
-        # copytree returns None in python2 and destination string in python3
-        # A error is raised in both cases, so we can just return 0 for success
         return 0
 
 
-Copy = ActionFactory(
-    copy_func,
-    lambda dest, src, symlinks=True: 'Copy("%s", "%s")' % (dest, src)
-)
+def copy_strfunc(dest, src, symlinks=True) -> str:
+    """strfunction for the Copy action function."""
+    return f'Copy({get_paths_str(dest)}, {get_paths_str(src)})'
+
+
+Copy = ActionFactory(copy_func, copy_strfunc)
 
 
-def delete_func(dest, must_exist=0):
+def delete_func(dest, must_exist=False) -> None:
+    """Implementation of the Delete action function.
+
+    Lets the Python :func:`os.unlink` raise an error if *dest* does not exist,
+    unless *must_exist* evaluates false (the default).
+    """
     SCons.Node.FS.invalidate_node_memos(dest)
     if not SCons.Util.is_List(dest):
         dest = [dest]
@@ -296,14 +337,16 @@ def delete_func(dest, must_exist=0):
         os.unlink(entry)
 
 
-def delete_strfunc(dest, must_exist=0):
-    return 'Delete(%s)' % get_paths_str(dest)
+def delete_strfunc(dest, must_exist=False) -> str:
+    """strfunction for the Delete action function."""
+    return f'Delete({get_paths_str(dest)})'
 
 
 Delete = ActionFactory(delete_func, delete_strfunc)
 
 
-def mkdir_func(dest):
+def mkdir_func(dest) -> None:
+    """Implementation of the Mkdir action function."""
     SCons.Node.FS.invalidate_node_memos(dest)
     if not SCons.Util.is_List(dest):
         dest = [dest]
@@ -311,22 +354,23 @@ def mkdir_func(dest):
         os.makedirs(str(entry), exist_ok=True)
 
 
-Mkdir = ActionFactory(mkdir_func,
-                      lambda _dir: 'Mkdir(%s)' % get_paths_str(_dir))
+Mkdir = ActionFactory(mkdir_func, lambda _dir: f'Mkdir({get_paths_str(_dir)})')
 
 
-def move_func(dest, src):
+def move_func(dest, src) -> None:
+    """Implementation of the Move action function."""
     SCons.Node.FS.invalidate_node_memos(dest)
     SCons.Node.FS.invalidate_node_memos(src)
     shutil.move(src, dest)
 
 
-Move = ActionFactory(move_func,
-                     lambda dest, src: 'Move("%s", "%s")' % (dest, src),
-                     convert=str)
+Move = ActionFactory(
+    move_func, lambda dest, src: f'Move("{dest}", "{src}")', convert=str
+)
 
 
-def touch_func(dest):
+def touch_func(dest) -> None:
+    """Implementation of the Touch action function."""
     SCons.Node.FS.invalidate_node_memos(dest)
     if not SCons.Util.is_List(dest):
         dest = [dest]
@@ -341,8 +385,7 @@ def touch_func(dest):
         os.utime(file, (atime, mtime))
 
 
-Touch = ActionFactory(touch_func,
-                      lambda file: 'Touch(%s)' % get_paths_str(file))
+Touch = ActionFactory(touch_func, lambda file: f'Touch({get_paths_str(file)})')
 
 
 # Internal utility functions


Test patch:
diff --git a/test/Chmod.py b/test/Chmod.py
--- a/test/Chmod.py
+++ b/test/Chmod.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 #
-# __COPYRIGHT__
+# MIT License
+#
+# Copyright The SCons Foundation
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -20,9 +22,6 @@
 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 """
 Verify that the Chmod() Action works.
@@ -43,26 +42,36 @@
 Execute(Chmod(('f1-File'), 0o666))
 Execute(Chmod('d2', 0o777))
 Execute(Chmod(Dir('d2-Dir'), 0o777))
+
 def cat(env, source, target):
     target = str(target[0])
     with open(target, "wb") as f:
         for src in source:
             with open(str(src), "rb") as infp:
                 f.write(infp.read())
+
 Cat = Action(cat)
 env = Environment()
-env.Command('bar.out', 'bar.in', [Cat,
-                                  Chmod("f3", 0o666),
-                                  Chmod("d4", 0o777)])
-env = Environment(FILE = 'f5')
+env.Command(
+    'bar.out',
+    'bar.in',
+    [Cat, Chmod("f3", 0o666), Chmod("d4", 0o777)],
+)
+env = Environment(FILE='f5')
 env.Command('f6.out', 'f6.in', [Chmod('$FILE', 0o666), Cat])
-env.Command('f7.out', 'f7.in', [Cat,
-                                Chmod('Chmod-$SOURCE', 0o666),
-                                Chmod('${TARGET}-Chmod', 0o666)])
+env.Command(
+    'f7.out',
+    'f7.in',
+    [Cat, Chmod('Chmod-$SOURCE', 0o666), Chmod('${TARGET}-Chmod', 0o666)],
+)
 
 # Make sure Chmod works with a list of arguments
-env = Environment(FILE = 'f9')
-env.Command('f8.out', 'f8.in', [Chmod(['$FILE', File('f10')], 0o666), Cat])
+env = Environment(FILE='f9')
+env.Command(
+    'f8.out',
+    'f8.in',
+    [Chmod(['$FILE', File('f10')], 0o666), Cat],
+)
 Execute(Chmod(['d11', Dir('d12')], 0o777))
 Execute(Chmod('f13', "a=r"))
 Execute(Chmod('f14', "ogu+w"))
@@ -117,28 +126,30 @@ def cat(env, source, target):
 os.chmod(test.workpath('d16'), 0o555)
 os.chmod(test.workpath('d17'), 0o555)
 os.chmod(test.workpath('d18'), 0o555)
-expect = test.wrap_stdout(read_str = """\
-Chmod("f1", 0666)
-Chmod("f1-File", 0666)
-Chmod("d2", 0777)
-Chmod("d2-Dir", 0777)
-Chmod(["d11", "d12"], 0777)
+
+expect = test.wrap_stdout(
+    read_str = """\
+Chmod("f1", 0o666)
+Chmod("f1-File", 0o666)
+Chmod("d2", 0o777)
+Chmod("d2-Dir", 0o777)
+Chmod(["d11", "d12"], 0o777)
 Chmod("f13", "a=r")
 Chmod("f14", "ogu+w")
 Chmod("f15", "ug=rw, go+ rw")
 Chmod("d16", "0777")
 Chmod(["d17", "d18"], "ogu = rwx")
 """,
-                          build_str = """\
+    build_str = """\
 cat(["bar.out"], ["bar.in"])
-Chmod("f3", 0666)
-Chmod("d4", 0777)
-Chmod("f5", 0666)
+Chmod("f3", 0o666)
+Chmod("d4", 0o777)
+Chmod("f5", 0o666)
 cat(["f6.out"], ["f6.in"])
 cat(["f7.out"], ["f7.in"])
-Chmod("Chmod-f7.in", 0666)
-Chmod("f7.out-Chmod", 0666)
-Chmod(["f9", "f10"], 0666)
+Chmod("Chmod-f7.in", 0o666)
+Chmod("f7.out-Chmod", 0o666)
+Chmod(["f9", "f10"], 0o666)
 cat(["f8.out"], ["f8.in"])
 """)
 test.run(options = '-n', arguments = '.', stdout = expect)
diff --git a/test/Copy-Action.py b/test/Copy-Action.py
--- a/test/Copy-Action.py
+++ b/test/Copy-Action.py
@@ -40,6 +40,10 @@
 Execute(Copy('f1.out', 'f1.in'))
 Execute(Copy(File('d2.out'), 'd2.in'))
 Execute(Copy('d3.out', File('f3.in')))
+# Issue #3009: make sure it's not mangled if src is a list.
+# make sure both list-of-str and list-of-Node work
+Execute(Copy('d7.out', ['f10.in', 'f11.in']))
+Execute(Copy('d7.out', Glob('f?.in')))
 
 def cat(env, source, target):
     target = str(target[0])
@@ -71,6 +75,7 @@ def cat(env, source, target):
 test.write(['d2.in', 'file'], "d2.in/file\n")
 test.write('f3.in', "f3.in\n")
 test.subdir('d3.out')
+test.subdir('d7.out')
 test.write('bar.in', "bar.in\n")
 test.write('f4.in', "f4.in\n")
 test.subdir('d5.in')
@@ -101,6 +106,8 @@ def cat(env, source, target):
 Copy("f1.out", "f1.in")
 Copy("d2.out", "d2.in")
 Copy("d3.out", "f3.in")
+Copy("d7.out", ["f10.in", "f11.in"])
+Copy("d7.out", ["f1.in", "f3.in", "f4.in", "f6.in", "f7.in", "f8.in", "f9.in"])
 """,
     build_str="""\
 cat(["bar.out"], ["bar.in"])
@@ -123,6 +130,8 @@ def cat(env, source, target):
 test.must_not_exist('f1.out')
 test.must_not_exist('d2.out')
 test.must_not_exist(os.path.join('d3.out', 'f3.in'))
+test.must_not_exist(os.path.join('d7.out', 'f7.in'))
+test.must_not_exist(os.path.join('d7.out', 'f11.in'))
 test.must_not_exist('f4.out')
 test.must_not_exist('d5.out')
 test.must_not_exist(os.path.join('d6.out', 'f6.in'))
@@ -141,6 +150,8 @@ def cat(env, source, target):
 test.must_match('f1.out', "f1.in\n", mode='r')
 test.must_match(['d2.out', 'file'], "d2.in/file\n", mode='r')
 test.must_match(['d3.out', 'f3.in'], "f3.in\n", mode='r')
+test.must_match(['d7.out', 'f7.in'], "f7.in\n", mode='r')
+test.must_match(['d7.out', 'f11.in'], "f11.in\n", mode='r')
 test.must_match('f4.out', "f4.in\n", mode='r')
 test.must_match(['d5.out', 'file'], "d5.in/file\n", mode='r')
 test.must_match(['d6.out', 'f6.in'], "f6.in\n", mode='r')