Repository: takluyver/flit
Pull request #136 (2017-10-08 14:16:15)

First steps towards reproducible sdist
This doesn't make sdists properly reproducible, but if you set `SOURCE_DATE_EPOCH` and build twice on the same machine (so the files have the same mtimes and permissions), you can get the same result.

CC @Carreau - we were talking about this somewhere else. I think this is all that's necessary to set the gzip timestamp:

```python
source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', '')
mtime = int(source_date_epoch) if source_date_epoch else None
gz = GzipFile(str(target), mode='wb', mtime=mtime)
tf = tarfile.TarFile(str(target), mode='w', fileobj=gz)
```


Patch:
diff --git a/flit/common.py b/flit/common.py
index 050f8eb..fcc82a5 100644
--- a/flit/common.py
+++ b/flit/common.py
@@ -153,6 +153,19 @@ def hash_file(path, algorithm='sha256'):
         h = hashlib.new(algorithm, f.read())
     return h.hexdigest()
 
+def normalize_file_permissions(st_mode):
+    """Normalize the permission bits in the st_mode field from stat to 644/755
+
+    Popular VCSs only track whether a file is executable or not. The exact
+    permissions can vary on systems with different umasks. Normalising
+    to 644 (non executable) or 755 (executable) makes builds more reproducible.
+    """
+    # Set 644 permissions, leaving higher bits of st_mode unchanged
+    new_mode = (st_mode | 0o644) & ~0o133
+    if st_mode & 0o100:
+        new_mode |= 0o111  # Executable: 644 -> 755
+    return new_mode
+
 class Metadata:
 
     home_page = None
diff --git a/flit/sdist.py b/flit/sdist.py
index 768135f..232dff7 100644
--- a/flit/sdist.py
+++ b/flit/sdist.py
@@ -1,5 +1,7 @@
 from configparser import ConfigParser
 from collections import defaultdict
+from copy import copy
+from gzip import GzipFile
 import io
 import logging
 import os
@@ -123,6 +125,25 @@ def include_path(p):
                 or (os.sep+'__pycache__' in p)
                 or p.endswith('.pyc'))
 
+def clean_tarinfo(ti, mtime=None):
+    """Clean metadata from a TarInfo object to make it more reproducible.
+
+    - Set uid & gid to 0
+    - Set uname and gname to ""
+    - Normalise permissions to 644 or 755
+    - Set mtime if not None
+    """
+    ti = copy(ti)
+    ti.uid = 0
+    ti.gid = 0
+    ti.uname = ''
+    ti.gname = ''
+    ti.mode = common.normalize_file_permissions(ti.mode)
+    if mtime is not None:
+        ti.mtime = mtime
+    return ti
+
+
 class SdistBuilder:
     def __init__(self, ini_path=Path('flit.ini')):
         self.ini_path = ini_path
@@ -209,37 +230,51 @@ class SdistBuilder:
             target_dir.mkdir(parents=True)
         target = target_dir / '{}-{}.tar.gz'.format(
                         self.metadata.name, self.metadata.version)
-        tf = tarfile.open(str(target), mode='w:gz')
-        tf_dir = '{}-{}'.format(self.metadata.name, self.metadata.version)
-
-        files_to_add = self.find_tracked_files()
-
-        for relpath in files_to_add:
-            path = self.srcdir / relpath
-            tf.add(str(path), arcname=pjoin(tf_dir, relpath))
-
-        if 'setup.py' in files_to_add:
-            log.warning("Using setup.py from repository, not generating setup.py")
-        else:
-            setup_py = self.make_setup_py()
-            log.info("Writing generated setup.py")
-            ti = tarfile.TarInfo(pjoin(tf_dir, 'setup.py'))
-            ti.size = len(setup_py)
-            tf.addfile(ti, io.BytesIO(setup_py))
-
-        pkg_info = PKG_INFO.format(
-            name=self.metadata.name,
-            version=self.metadata.version,
-            summary=self.metadata.summary,
-            home_page=self.metadata.home_page,
-            author=self.metadata.author,
-            author_email=self.metadata.author_email,
-        ).encode('utf-8')
-        ti = tarfile.TarInfo(pjoin(tf_dir, 'PKG-INFO'))
-        ti.size = len(pkg_info)
-        tf.addfile(ti, io.BytesIO(pkg_info))
-
-        tf.close()
+        source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', '')
+        mtime = int(source_date_epoch) if source_date_epoch else None
+        gz = GzipFile(str(target), mode='wb', mtime=mtime)
+        tf = tarfile.TarFile(str(target), mode='w', fileobj=gz)
+
+        try:
+            tf_dir = '{}-{}'.format(self.metadata.name, self.metadata.version)
+
+            files_to_add = self.find_tracked_files()
+
+            for relpath in files_to_add:
+                path = self.srcdir / relpath
+                ti = tf.gettarinfo(str(path), arcname=pjoin(tf_dir, relpath))
+                ti = clean_tarinfo(ti, mtime)
+
+                if ti.isreg():
+                    with path.open('rb') as f:
+                        tf.addfile(ti, f)
+                else:
+                    tf.addfile(ti)  # Symlinks & ?
+
+            if 'setup.py' in files_to_add:
+                log.warning("Using setup.py from repository, not generating setup.py")
+            else:
+                setup_py = self.make_setup_py()
+                log.info("Writing generated setup.py")
+                ti = tarfile.TarInfo(pjoin(tf_dir, 'setup.py'))
+                ti.size = len(setup_py)
+                tf.addfile(ti, io.BytesIO(setup_py))
+
+            pkg_info = PKG_INFO.format(
+                name=self.metadata.name,
+                version=self.metadata.version,
+                summary=self.metadata.summary,
+                home_page=self.metadata.home_page,
+                author=self.metadata.author,
+                author_email=self.metadata.author_email,
+            ).encode('utf-8')
+            ti = tarfile.TarInfo(pjoin(tf_dir, 'PKG-INFO'))
+            ti.size = len(pkg_info)
+            tf.addfile(ti, io.BytesIO(pkg_info))
+
+        finally:
+            tf.close()
+            gz.close()
 
         log.info("Built sdist: %s", target)
         return target
diff --git a/flit/wheel.py b/flit/wheel.py
index f6d0194..ef1d6da 100644
--- a/flit/wheel.py
+++ b/flit/wheel.py
@@ -93,10 +93,7 @@ class WheelBuilder:
         
         # Normalize permission bits to either 755 (executable) or 644
         st_mode = os.stat(full_path).st_mode
-        new_mode = (st_mode | 0o644) & ~0o133  # 644 permissions, higher bits unchanged
-        if st_mode & 0o100:
-            # If executable, 644 -> 755
-            new_mode |= 0o111
+        new_mode = common.normalize_file_permissions(st_mode)
         zinfo.external_attr = (new_mode & 0xFFFF) << 16      # Unix attributes
 
         if stat.S_ISDIR(st_mode):


Test patch:
diff --git a/tests/test_common.py b/tests/test_common.py
index 7c3224f..73c5fe8 100644
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -3,7 +3,9 @@ from pathlib import Path
 from unittest import TestCase
 import pytest
 
-from flit.common import Module, get_info_from_module, InvalidVersion, check_version
+from flit.common import (Module, get_info_from_module, InvalidVersion,
+     check_version, normalize_file_permissions
+)
 
 samples_dir = os.path.join(os.path.dirname(__file__), 'samples')
 
@@ -49,4 +51,6 @@ class ModuleTests(TestCase):
         assert check_version('4.1.0b1') == True
         assert check_version('4.1.0beta1') == False
 
-
+def test_normalize_file_permissions():
+    assert normalize_file_permissions(0o100664) == 0o100644 # regular file
+    assert normalize_file_permissions(0o40775) == 0o40755   # directory
diff --git a/tests/test_sdist.py b/tests/test_sdist.py
index fd6cdfb..be55d06 100644
--- a/tests/test_sdist.py
+++ b/tests/test_sdist.py
@@ -1,9 +1,11 @@
 import ast
+from io import BytesIO
 from os.path import join as pjoin
 from pathlib import Path
 import pytest
 from shutil import which, copy
 import sys
+import tarfile
 from tempfile import TemporaryDirectory
 from testpath import assert_isfile, MockCommand
 
@@ -86,3 +88,11 @@ def test_make_setup_py():
     assert 'install_requires' not in ns
     assert ns['entry_points'] == \
            {'console_scripts': ['pkg_script = package1:main']}
+
+def test_clean_tarinfo():
+    with tarfile.open(mode='w', fileobj=BytesIO()) as tf:
+        ti = tf.gettarinfo(str(samples_dir / 'module1.py'))
+    cleaned = sdist.clean_tarinfo(ti, mtime=42)
+    assert cleaned.uid == 0
+    assert cleaned.uname == ''
+    assert cleaned.mtime == 42