Repository: pex-tool/pex

Issue:
Reproducibility of `pex-tools extract`
`pex-tools $file extract -f $dest` currently creates output wheels with internal timestamps tied to their creation time, rather than zeroed or cloned from the producing PEX.


Discussion:
> or cloned 

Since a PEX does not contain wheels but, instead, installed wheels, the wheels produced by extract cannot be cloned.

Ah - you probably meant cloning the wheel timestamps - that could be done via the `SOURCE_DATE_EPOCH` trick linked below.
The fix is exporting `SOURCE_DATE_EPOCH` when packing the wheel:
https://github.com/pypa/wheel/blob/41b375ef33e3368f6f58c13f9649491d01c2ba5e/src/wheel/wheelfile.py#L24-L28

Resolving patch:
diff --git a/pex/common.py b/pex/common.py
--- a/pex/common.py
+++ b/pex/common.py
@@ -41,6 +41,8 @@
 DETERMINISTIC_DATETIME = datetime(
     year=1980, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None
 )
+_UNIX_EPOCH = datetime(year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None)
+DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds()
 
 
 def filter_pyc_dirs(dirs):
@@ -306,7 +308,7 @@ def safe_rmtree(directory):
 
 
 def safe_sleep(seconds):
-    # type: (int) -> None
+    # type: (float) -> None
     """Ensure that the thread sleeps at a minimum the requested seconds.
 
     Until Python 3.5, there was no guarantee that time.sleep() would actually sleep the requested
diff --git a/pex/tools/commands/repository.py b/pex/tools/commands/repository.py
--- a/pex/tools/commands/repository.py
+++ b/pex/tools/commands/repository.py
@@ -15,7 +15,14 @@
 from threading import Thread
 
 from pex import dist_metadata, pex_warnings
-from pex.common import pluralize, safe_mkdir, safe_mkdtemp, safe_open
+from pex.common import (
+    DETERMINISTIC_DATETIME,
+    DETERMINISTIC_DATETIME_TIMESTAMP,
+    pluralize,
+    safe_mkdir,
+    safe_mkdtemp,
+    safe_open,
+)
 from pex.compatibility import Queue
 from pex.environment import PEXEnvironment
 from pex.interpreter import PythonIdentity, PythonInterpreter, spawn_python_job
@@ -127,6 +134,20 @@ def _add_extract_arguments(subparsers):
             action="store_true",
             help="Also extract a wheel for the PEX file sources.",
         )
+        extract_parser.add_argument(
+            "--use-system-time",
+            dest="use_system_time",
+            default=False,
+            action="store_true",
+            help=(
+                "Use the current system time to generate timestamps for the extracted "
+                "distributions. Otherwise, Pex will use midnight on January 1, 1980. By using "
+                "system time, the extracted distributions will not be reproducible, meaning that "
+                "if you were to re-run extraction against the same PEX file then the newly "
+                "extracted distributions would not be byte-for-byte identical distributions "
+                "extracted in prior runs."
+            ),
+        )
         extract_parser.add_argument(
             "--serve",
             action="store_true",
@@ -233,11 +254,20 @@ def _extract(
 
         def spawn_extract(distribution):
             # type: (Distribution) -> SpawnedJob[Text]
+            env = os.environ.copy()
+            if not options.use_system_time:
+                # N.B.: The `SOURCE_DATE_EPOCH` env var is semi-standard magic for controlling
+                # build tools. Wheel has supported this since 2016.
+                # See:
+                # + https://reproducible-builds.org/docs/source-date-epoch/
+                # + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39
+                env.update(SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP)))
             job = spawn_python_job(
                 args=["-m", "wheel", "pack", "--dest-dir", dest_dir, distribution.location],
                 interpreter=pex.interpreter,
                 expose=["wheel"],
                 stdout=subprocess.PIPE,
+                env=env,
             )
             return SpawnedJob.stdout(
                 job, result_func=lambda out: "{}: {}".format(distribution, out.decode())


Test patch:
diff --git a/pex/testing.py b/pex/testing.py
--- a/pex/testing.py
+++ b/pex/testing.py
@@ -12,7 +12,15 @@
 from contextlib import contextmanager
 from textwrap import dedent
 
-from pex.common import atomic_directory, open_zip, safe_mkdir, safe_mkdtemp, temporary_dir
+from pex.common import (
+    atomic_directory,
+    open_zip,
+    safe_mkdir,
+    safe_mkdtemp,
+    safe_rmtree,
+    safe_sleep,
+    temporary_dir,
+)
 from pex.compatibility import to_unicode
 from pex.distribution_target import DistributionTarget
 from pex.executor import Executor
@@ -344,18 +352,34 @@ def assert_failure(self):
         assert self.return_code != 0
 
 
-def run_pex_command(args, env=None, python=None, quiet=False):
-    # type: (Iterable[str], Optional[Dict[str, str]], Optional[str], bool) -> IntegResults
+def create_pex_command(
+    args=None,  # type: Optional[Iterable[str]]
+    python=None,  # type: Optional[str]
+    quiet=False,  # type: bool
+):
+    # type: (...) -> List[str]
+    cmd = [python or sys.executable, "-mpex"]
+    if not quiet:
+        cmd.append("-vvvvv")
+    if args:
+        cmd.extend(args)
+    return cmd
+
+
+def run_pex_command(
+    args,  # type: Iterable[str]
+    env=None,  # type: Optional[Dict[str, str]]
+    python=None,  # type: Optional[str]
+    quiet=False,  # type: bool
+):
+    # type: (...) -> IntegResults
     """Simulate running pex command for integration testing.
 
     This is different from run_simple_pex in that it calls the pex command rather than running a
     generated pex.  This is useful for testing end to end runs with specific command line arguments
     or env options.
     """
-    cmd = [python or sys.executable, "-mpex"]
-    if not quiet:
-        cmd.append("-vvvvv")
-    cmd.extend(args)
+    cmd = create_pex_command(args, python=python, quiet=quiet)
     process = Executor.open_process(
         cmd=cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE
     )
@@ -540,3 +564,64 @@ def pushd(directory):
         yield
     finally:
         os.chdir(cwd)
+
+
+def make_env(**kwargs):
+    # type: (**Any) -> Dict[str, str]
+    """Create a copy of the current environment with the given modifications.
+
+    The given kwargs add to or update the environment when they have a non-`None` value. When they
+    have a `None` value, the environment variable is removed from the environment.
+
+    All non-`None` values are converted to strings by apply `str`.
+    """
+    env = os.environ.copy()
+    env.update((k, str(v)) for k, v in kwargs.items() if v is not None)
+    for k, v in kwargs.items():
+        if v is None:
+            env.pop(k, None)
+    return env
+
+
+def run_command_with_jitter(
+    args,  # type: Iterable[str]
+    path_argument,  # type: str
+    extra_env=None,  # type: Optional[Mapping[str, str]]
+    python=None,  # type: Optional[str]
+    delay=2.0,  # type: float
+    count=3,  # type: int
+):
+    # type: (...) -> List[str]
+    """Runs the command `count` times in an attempt to introduce randomness.
+
+    Each run of the command will run against a clean Pex cache with a unique path injected as the
+    value for `path_argument`. A unique `PYTHONHASHSEED` is set in the environment for each
+    execution as well.
+
+    Additionally, a delay is inserted between executions. By default, this delay is 2s to ensure zip
+    precision is stressed. See: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT.
+    """
+    td = safe_mkdtemp()
+    pex_root = os.path.join(td, "pex_root")
+
+    paths = []
+    for index in range(count):
+        path = os.path.join(td, str(index))
+        cmd = list(args) + [path_argument, path]
+
+        # Note that we change the `PYTHONHASHSEED` to ensure that there are no issues resulting
+        # from the random seed, such as data structures, as Tox sets this value by default.
+        # See:
+        # https://tox.readthedocs.io/en/latest/example/basic.html#special-handling-of-pythonhashseed
+        env = make_env(PEX_ROOT=pex_root, PYTHONHASHSEED=(index * 497) + 4)
+        if extra_env:
+            env.update(extra_env)
+
+        if index > 0:
+            safe_sleep(delay)
+
+        # Ensure the PEX is fully rebuilt.
+        safe_rmtree(pex_root)
+        subprocess.check_call(args=cmd, env=env)
+        paths.append(path)
+    return paths
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -7,7 +7,6 @@
 import filecmp
 import functools
 import glob
-import itertools
 import json
 import multiprocessing
 import os
@@ -23,15 +22,7 @@
 
 import pytest
 
-from pex.common import (
-    safe_copy,
-    safe_mkdir,
-    safe_open,
-    safe_rmtree,
-    safe_sleep,
-    temporary_dir,
-    touch,
-)
+from pex.common import safe_copy, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch
 from pex.compatibility import WINDOWS, to_bytes
 from pex.executor import Executor
 from pex.fetcher import URLFetcher
@@ -53,10 +44,13 @@
     IntegResults,
     WheelBuilder,
     built_wheel,
+    create_pex_command,
     ensure_python_interpreter,
     ensure_python_venv,
     get_dep_dist_names_from_pex,
+    make_env,
     make_source_dir,
+    run_command_with_jitter,
     run_pex_command,
     run_simple_pex,
     run_simple_pex_test,
@@ -84,16 +78,6 @@
     )
 
 
-def make_env(**kwargs):
-    # type: (**Any) -> Dict[str, str]
-    env = os.environ.copy()
-    env.update((k, str(v)) for k, v in kwargs.items() if v is not None)
-    for k, v in kwargs.items():
-        if v is None:
-            env.pop(k, None)
-    return env
-
-
 def test_pex_execute():
     # type: () -> None
     body = "print('Hello')"
@@ -1773,31 +1757,6 @@ def test_issues_539_abi3_resolution():
 def assert_reproducible_build(args):
     # type: (List[str]) -> None
     with temporary_dir() as td:
-        pex_root = os.path.join(td, "pex_root")
-
-        # Note that we change the `PYTHONHASHSEED` to ensure that there are no issues resulting
-        # from the random seed, such as data structures, as Tox sets this value by default. See
-        # https://tox.readthedocs.io/en/latest/example/basic.html#special-handling-of-pythonhashseed.
-        def create_pex(path, seed):
-            safe_rmtree(pex_root)
-            result = run_pex_command(
-                args + ["-o", path, "--pex-root", pex_root], env=make_env(PYTHONHASHSEED=seed)
-            )
-            result.assert_success()
-
-        def create_multiple_pexes():
-            paths = []
-            for index in range(3):
-                path = os.path.join(td, "{}.pex".format(index))
-                paths.append(path)
-                # We sleep to ensure that there is no non-reproducibility from timestamps or
-                # anything that may depend on the system time. Note that we must sleep for at least
-                # 2 seconds, because the zip format uses 2 second precision per section 4.4.6 of
-                # https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT.
-                if index > 0:
-                    safe_sleep(2)
-                create_pex(path, seed=(index * 497) + 4)
-            return paths
 
         def explode_pex(path):
             with ZipFile(path) as zf:
@@ -1806,10 +1765,12 @@ def explode_pex(path):
                 zf.extractall(path=destination_dir)
                 return [os.path.join(destination_dir, member) for member in sorted(zf.namelist())]
 
-        pexes = create_multiple_pexes()
+        pexes = run_command_with_jitter(
+            path_argument="--output-file", args=create_pex_command(quiet=True), count=3
+        )
         pex_members = {pex: explode_pex(path=pex) for pex in pexes}
-
-        for pex1, pex2 in itertools.combinations(pexes, r=2):
+        pex1 = pexes.pop()
+        for pex2 in pexes:
             # First compare file-by-file for easier debugging.
             for member1, member2 in zip(pex_members[pex1], pex_members[pex2]):
                 assert not os.path.isdir(member1) ^ os.path.isdir(member2)
diff --git a/tests/tools/commands/test_repository.py b/tests/tools/commands/test_repository.py
--- a/tests/tools/commands/test_repository.py
+++ b/tests/tools/commands/test_repository.py
@@ -3,16 +3,19 @@
 
 from __future__ import absolute_import
 
+import filecmp
+import itertools
 import json
 import os
 import signal
 import subprocess
+import time
 from textwrap import dedent
 
 import pytest
 
-from pex.common import safe_open, temporary_dir
-from pex.testing import PY38, ensure_python_venv, run_pex_command
+from pex.common import DETERMINISTIC_DATETIME, open_zip, safe_open, temporary_dir
+from pex.testing import PY38, ensure_python_venv, run_command_with_jitter, run_pex_command
 from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.pkg_resources import Distribution, Requirement
 from pex.typing import TYPE_CHECKING
@@ -128,7 +131,83 @@ def test_info_verbose(pex, pex_tools_env):
     } == {Requirement.parse(req) for req in requests_info["requires_dists"]}
 
 
-def test_extract(pex, pex_tools_env, tmpdir):
+def test_extract_deterministic_timestamp(pex, pex_tools_env, tmpdir):
+    deterministic_date_time = (
+        DETERMINISTIC_DATETIME.year,
+        DETERMINISTIC_DATETIME.month,
+        DETERMINISTIC_DATETIME.day,
+        DETERMINISTIC_DATETIME.hour,
+        DETERMINISTIC_DATETIME.minute,
+        DETERMINISTIC_DATETIME.second,
+    )
+
+    deterministic_dists_dir = os.path.join(str(tmpdir), "deterministic-dists")
+    subprocess.check_call(
+        args=[pex, "repository", "extract", "-f", deterministic_dists_dir], env=pex_tools_env
+    )
+    with open_zip(
+        os.path.join(deterministic_dists_dir, "requests-2.25.1-py2.py3-none-any.whl")
+    ) as zipfile:
+        infolist = zipfile.infolist()
+        assert len(infolist) > 0
+        for info in infolist:
+            assert deterministic_date_time == info.date_time
+
+    non_deterministic_dists_dir = os.path.join(str(tmpdir), "non_deterministic_dists_dir")
+    subprocess.check_call(
+        args=[pex, "repository", "extract", "-f", non_deterministic_dists_dir, "--use-system-time"],
+        env=pex_tools_env,
+    )
+    with open_zip(
+        os.path.join(non_deterministic_dists_dir, "requests-2.25.1-py2.py3-none-any.whl")
+    ) as zipfile:
+        infolist = zipfile.infolist()
+        assert len(infolist) > 0
+        for info in infolist:
+            assert deterministic_date_time != info.date_time
+
+
+def test_extract_non_deterministic_wheels(pex, pex_tools_env):
+    dist_dirs = run_command_with_jitter(
+        args=[pex, "repository", "extract", "--use-system-time"],
+        path_argument="-f",
+        extra_env=pex_tools_env,
+        count=3,
+    )
+    for dists_dir1, dists_dir2 in itertools.combinations(dist_dirs, 2):
+        dists1 = sorted(os.listdir(dists_dir1))
+        assert len(dists1) > 1
+        assert dists1 == sorted(os.listdir(dists_dir2))
+
+        same, different, non_reg = filecmp.cmpfiles(
+            dists_dir1, dists_dir2, common=dists1, shallow=False
+        )
+        assert not same
+        assert not non_reg
+        assert sorted(dists1) == sorted(different)
+
+
+def test_extract_deterministic_wheels(pex, pex_tools_env):
+    # We already have tests that ensure PEX file creation is deterministic; so here we just test
+    # that extracting wheels from a fixed PEX file is also deterministic.
+
+    dist_dirs = run_command_with_jitter(
+        args=[pex, "repository", "extract"], path_argument="-f", extra_env=pex_tools_env, count=3
+    )
+    dists_dir1 = dist_dirs.pop()
+    dists1 = sorted(os.listdir(dists_dir1))
+    assert len(dists1) > 1
+    for dists_dir2 in dist_dirs:
+        assert dists1 == sorted(os.listdir(dists_dir2))
+        same, different, non_reg = filecmp.cmpfiles(
+            dists_dir1, dists_dir2, common=dists1, shallow=False
+        )
+        assert not different
+        assert not non_reg
+        assert sorted(dists1) == sorted(same)
+
+
+def test_extract_lifecycle(pex, pex_tools_env, tmpdir):
     # type: (str, Dict[str, str], Any) -> None
     dists_dir = os.path.join(str(tmpdir), "dists")
     pid_file = os.path.join(str(tmpdir), "pid")