Repository: SCons/scons

Issue:
Add timestamp override to Zip builder
Zip should have an option to override the timestamp on files in the archive instead of taking it from the file system. This can be useful for repeatable builds or for anonymizing the archive.


Discussion:
I'm not sure we can do this when we use pure python zipfile unless the floor for this is Python 3.6
https://docs.python.org/3.6/library/zipfile.html#zipinfo-objects

Can you modify zipinfo in py 3.5?
Thoughts @dholth 

I was planning to submit an implementation when I get time,

Wouldn't it be okay for this to only work when Python > 3.6

I don't think any of my target audience uses 3.5

On Mon, Jan 11, 2021, at 6:54 PM, William Deegan wrote:
> 


> I'm not sure we can do this when we use pure python zipfile unless the floor for this is Python 3.6
> https://docs.python.org/3.6/library/zipfile.html#zipinfo-objects


> Can you modify zipinfo in py 3.5?
> Thoughts @dholth <https://github.com/dholth>


> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub <https://github.com/SCons/scons/issues/3843#issuecomment-758298391>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AABSZESFNLIFNY35XNSE7HDSZOFRTANCNFSM4U4ZM6VQ>.

not an official answer, but py35 is out of support so a >3.5 answer isn't a completely unusable one, just might take a bit to get integrated?
Historically SCons has supported all functionality on all supported versions of Python.
SCons currently supports Python 3.5+

Might make sense to just issue a warning that it's not supported when user is running Python < 3.6?


This works on py35+:

```python
def zip(target, source, env):
    files = []
    for s in source:
        if s.isdir():
            for dirpath, dirnames, filenames in os.walk(str(s)):
                for fname in filenames:
                    path = os.path.join(dirpath, fname)
                    if os.path.isfile(path):
                        files.append(path)
        else:
            files.append(str(s))
    
    compression = env.get('ZIPCOMPRESSION', 0)
    zip_root = str(env.get('ZIPROOT', ''))

    with zipfile.ZipFile(str(target[0]), 'w', compression) as zf:
        for fname in files:
            with open(fname, "rb") as f:
                arcname = os.path.relpath(fname, zip_root)
                zf.writestr(zipfile.ZipInfo(arcname, date_time=(1980,1,1,1,1,1)), f.read())
```
Probably better if that function gets renamed since `zip` is also a Python built-in function (I realize that's an existing decision, not introduced by @davidhart82 proposal)
@davidhart82 - would you be up for creating a PR.. you'd just need to add a test.
@bdbaddog sure thing, I have time this week for open source contributions. How do you want the date to be specified by the user? A string? With an optional format specifier? Or something simpler? 
@davidhart82 - how about ZIP_OVERRIDE_TIMESTAMP
Contents should be the tuple which wouldmap right to datetime?
https://docs.python.org/3.5/library/datetime.html#datetime-objects

Thoughts?
@bdbaddog sure, sounds good. Will try and get that done tomorrow. 

Resolving patch:
diff --git a/SCons/Tool/zip.py b/SCons/Tool/zip.py
--- a/SCons/Tool/zip.py
+++ b/SCons/Tool/zip.py
@@ -8,8 +8,9 @@
 
 """
 
+# MIT License
 #
-# __COPYRIGHT__
+# Copyright The SCons Foundation
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -29,39 +30,62 @@
 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
-import os.path
+import os
 
 import SCons.Builder
 import SCons.Defaults
 import SCons.Node.FS
 import SCons.Util
 
+import time
 import zipfile
 
+
 zip_compression = zipfile.ZIP_DEFLATED
 
 
-def zip(target, source, env):
-    compression = env.get('ZIPCOMPRESSION', 0)
-    zf = zipfile.ZipFile(str(target[0]), 'w', compression)
+def _create_zipinfo_for_file(fname, arcname, date_time, compression):
+    st = os.stat(fname)
+    if not date_time:
+        mtime = time.localtime(st.st_mtime)
+        date_time = mtime[0:6]
+    zinfo = zipfile.ZipInfo(filename=arcname, date_time=date_time)
+    zinfo.external_attr = (st.st_mode & 0xFFFF) << 16  # Unix attributes
+    zinfo.compress_type = compression
+    zinfo.file_size = st.st_size
+    return zinfo
+
+
+def zip_builder(target, source, env):
+    compression = env.get('ZIPCOMPRESSION', zipfile.ZIP_STORED)
+    zip_root = str(env.get('ZIPROOT', ''))
+    date_time = env.get('ZIP_OVERRIDE_TIMESTAMP')
+
+    files = []
     for s in source:
         if s.isdir():
             for dirpath, dirnames, filenames in os.walk(str(s)):
                 for fname in filenames:
                     path = os.path.join(dirpath, fname)
                     if os.path.isfile(path):
-                        zf.write(path, os.path.relpath(path, str(env.get('ZIPROOT', ''))))
+                        files.append(path)
         else:
-            zf.write(str(s), os.path.relpath(str(s), str(env.get('ZIPROOT', ''))))
-    zf.close()
+            files.append(str(s))
+
+    with zipfile.ZipFile(str(target[0]), 'w', compression) as zf:
+        for fname in files:
+            arcname = os.path.relpath(fname, zip_root)
+            # TODO: Switch to ZipInfo.from_file when 3.6 becomes the base python version
+            zinfo = _create_zipinfo_for_file(fname, arcname, date_time, compression)
+            with open(fname, "rb") as f:
+                zf.writestr(zinfo, f.read())
+
 
 # Fix PR #3569 - If you don't specify ZIPCOM and ZIPCOMSTR when creating
 # env, then it will ignore ZIPCOMSTR set afterwards.
-zipAction = SCons.Action.Action(zip, "$ZIPCOMSTR", varlist=['ZIPCOMPRESSION'])
+zipAction = SCons.Action.Action(zip_builder, "$ZIPCOMSTR",
+                                varlist=['ZIPCOMPRESSION', 'ZIPROOT', 'ZIP_OVERRIDE_TIMESTAMP'])
 
 ZipBuilder = SCons.Builder.Builder(action=SCons.Action.Action('$ZIPCOM', '$ZIPCOMSTR'),
                                    source_factory=SCons.Node.FS.Entry,


Test patch:
diff --git a/test/ZIP/ZIP.py b/test/ZIP/ZIP.py
--- a/test/ZIP/ZIP.py
+++ b/test/ZIP/ZIP.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 #
-# __COPYRIGHT__
+# MIT License
+#
+# Copyright The SCons Foundation
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -20,9 +22,6 @@
 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os
 import stat
@@ -33,30 +32,6 @@
 
 test = TestSCons.TestSCons()
 
-try:
-    import zipfile
-except ImportError:
-    x = "Python version has no 'ziplib' module; skipping tests.\n"
-    test.skip_test(x)
-
-def zipfile_contains(zipfilename, names):
-    """Returns True if zipfilename contains all the names, False otherwise."""
-    zf=zipfile.ZipFile(zipfilename, 'r')
-    if type(names)==type(''):
-        names=[names]
-    for name in names:
-        try:
-            info=zf.getinfo(name)
-        except KeyError as e:     # name not found
-            zf.close()
-            return False
-    return True
-
-def zipfile_files(fname):
-    """Returns all the filenames in zip file fname."""
-    zf = zipfile.ZipFile(fname, 'r')
-    return [x.filename for x in zf.infolist()]
-
 test.subdir('sub1')
 
 test.write('SConstruct', """
@@ -78,12 +53,12 @@ def zipfile_files(fname):
 test.run(arguments = 'aaa.zip', stderr = None)
 
 test.must_exist('aaa.zip')
-test.fail_test(not zipfile_contains('aaa.zip', ['file1', 'file2', 'file3']))
+test.fail_test(not test.zipfile_contains('aaa.zip', ['file1', 'file2', 'file3']))
 
 test.run(arguments = 'bbb.zip', stderr = None)
 
 test.must_exist('bbb.zip')
-test.fail_test(not zipfile_contains('bbb.zip', ['sub1/file5', 'sub1/file6', 'file4']))
+test.fail_test(not test.zipfile_contains('bbb.zip', ['sub1/file5', 'sub1/file6', 'file4']))
 
 ######
 
@@ -136,11 +111,11 @@ def marker(target, source, env):
 test.must_not_exist(test.workpath('f3.zip'))
 test.must_exist(test.workpath('f3.xyzzy'))
 
-test.fail_test(zipfile_files("f1.zip") != ['file10', 'file11', 'file12'])
+test.fail_test(test.zipfile_files("f1.zip") != ['file10', 'file11', 'file12'])
 
-test.fail_test(zipfile_files("f2.zip") != ['file13', 'file14', 'file15'])
+test.fail_test(test.zipfile_files("f2.zip") != ['file13', 'file14', 'file15'])
 
-test.fail_test(zipfile_files("f3.xyzzy") != ['file16', 'file17', 'file18'])
+test.fail_test(test.zipfile_files("f3.xyzzy") != ['file16', 'file17', 'file18'])
 
 f4_size = os.stat('f4.zip')[stat.ST_SIZE]
 f4stored_size = os.stat('f4stored.zip')[stat.ST_SIZE]
diff --git a/test/ZIP/ZIPROOT.py b/test/ZIP/ZIPROOT.py
--- a/test/ZIP/ZIPROOT.py
+++ b/test/ZIP/ZIPROOT.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 #
-# __COPYRIGHT__
+# MIT License
+#
+# Copyright The SCons Foundation
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -20,9 +22,6 @@
 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 
 import TestSCons
@@ -33,23 +32,6 @@
 
 import zipfile
 
-def zipfile_contains(zipfilename, names):
-    """Returns True if zipfilename contains all the names, False otherwise."""
-    zf=zipfile.ZipFile(zipfilename, 'r')
-    if type(names)==type(''):
-        names=[names]
-    for name in names:
-        try:
-            info=zf.getinfo(name)
-        except KeyError as e:     # name not found
-            zf.close()
-            return False
-    return True
-
-def zipfile_files(fname):
-    """Returns all the filenames in zip file fname."""
-    zf = zipfile.ZipFile(fname, 'r')
-    return [x.filename for x in zf.infolist()]
 
 test.subdir('sub1')
 test.subdir(['sub1', 'sub2'])
@@ -69,13 +51,12 @@ def zipfile_files(fname):
 test.must_exist('aaa.zip')
 
 # TEST: Zip file should contain 'file1', not 'sub1/file1', because of ZIPROOT.
-zf=zipfile.ZipFile('aaa.zip', 'r')
-test.fail_test(zf.testzip() is not None)
-zf.close()
+with zipfile.ZipFile('aaa.zip', 'r') as zf:
+    test.fail_test(zf.testzip() is not None)
 
-files=zipfile_files('aaa.zip')
-test.fail_test(zipfile_files('aaa.zip') != ['file1'],
-               message='Zip file aaa.zip has wrong files: %s'%repr(files))
+files = test.zipfile_files('aaa.zip')
+test.fail_test(test.zipfile_files('aaa.zip') != ['file1'],
+               message='Zip file aaa.zip has wrong files: %s' % repr(files))
 
 ###
 
@@ -84,13 +65,12 @@ def zipfile_files(fname):
 test.must_exist('bbb.zip')
 
 # TEST: Zip file should contain 'sub2/file2', not 'sub1/sub2/file2', because of ZIPROOT.
-zf=zipfile.ZipFile('bbb.zip', 'r')
-test.fail_test(zf.testzip() is not None)
-zf.close()
+with zipfile.ZipFile('bbb.zip', 'r') as zf:
+    test.fail_test(zf.testzip() is not None)
 
-files=zipfile_files('bbb.zip')
-test.fail_test(zipfile_files('bbb.zip') != ['file2', 'sub2/file2'],
-               message='Zip file bbb.zip has wrong files: %s'%repr(files))
+files = test.zipfile_files('bbb.zip')
+test.fail_test(test.zipfile_files('bbb.zip') != ['file2', 'sub2/file2'],
+               message='Zip file bbb.zip has wrong files: %s' % repr(files))
 
 
 test.pass_test()
diff --git a/test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py b/test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py
new file mode 100644
--- /dev/null
+++ b/test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+#
+# MIT License
+#
+# Copyright The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+import TestSCons
+
+_python_ = TestSCons._python_
+
+test = TestSCons.TestSCons()
+
+import zipfile
+
+
+def zipfile_get_file_datetime(zipfilename, fname):
+    """Returns the date_time of the given file with the archive."""
+    with zipfile.ZipFile(zipfilename, 'r') as zf:
+        for info in zf.infolist():
+            if info.filename == fname:
+                return info.date_time
+    
+    raise Exception("Unable to find %s" % fname)
+
+
+test.write('SConstruct', """
+env = Environment(tools = ['zip'])
+env.Zip(target = 'aaa.zip', source = ['file1'], ZIP_OVERRIDE_TIMESTAMP=(1983,3,11,1,2,2))
+""" % locals())
+
+test.write(['file1'], "file1\n")
+
+test.run(arguments = 'aaa.zip', stderr = None)
+
+test.must_exist('aaa.zip')
+
+with zipfile.ZipFile('aaa.zip', 'r') as zf:
+    test.fail_test(zf.testzip() is not None)
+
+files = test.zipfile_files('aaa.zip')
+test.fail_test(files != ['file1'],
+               message='Zip file aaa.zip has wrong files: %s' % repr(files))
+
+date_time = zipfile_get_file_datetime('aaa.zip', 'file1')
+test.fail_test(date_time != (1983, 3, 11, 1, 2, 2),
+               message="Zip file aaa.zip#file1 has wrong date_time: %s" % repr(date_time))
+
+test.pass_test()
diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py
--- a/testing/framework/TestSCons.py
+++ b/testing/framework/TestSCons.py
@@ -12,9 +12,28 @@
 attributes defined in this subclass.
 """
 
-# __COPYRIGHT__
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+# MIT License
+#
+# Copyright The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import os
 import re
@@ -22,6 +41,7 @@
 import sys
 import time
 import subprocess
+import zipfile
 from collections import namedtuple
 
 from TestCommon import *
@@ -1604,6 +1624,16 @@ def platform_has_symlink(self):
         else:
             return True
 
+    def zipfile_contains(self, zipfilename, names):
+        """Returns True if zipfilename contains all the names, False otherwise."""
+        with zipfile.ZipFile(zipfilename, 'r') as zf:
+            return all(elem in zf.namelist() for elem in names)
+
+    def zipfile_files(self, fname):
+        """Returns all the filenames in zip file fname."""
+        with zipfile.ZipFile(fname, 'r') as zf:
+            return zf.namelist()
+
 
 class Stat:
     def __init__(self, name, units, expression, convert=None):