Repository: chriskuehl/lazy-build

Issue:
gzip before upload
In some common cases (e.g. node_modules), this reduces ~300MB artifacts to ~100MB artifacts. Probably worth the CPU time.

Discussion:

Patch:
diff --git a/lazy_build/cache.py b/lazy_build/cache.py
index 27124d9..f1ec74a 100644
--- a/lazy_build/cache.py
+++ b/lazy_build/cache.py
@@ -20,10 +20,15 @@ class S3Backend(collections.namedtuple('S3Backend', (
     def key_for_ctx(self, ctx):
         return self.path.rstrip('/') + '/' + ctx.hash
 
+    def artifact_paths(self, ctx):
+        key = self.key_for_ctx(ctx)
+        return key + '.tar.gz', key + '.json'
+
     def has_artifact(self, ctx):
+        tarball, json = self.artifact_paths(ctx)
         # what a ridiculous dance we have to do here...
         try:
-            self.s3.Object(self.bucket, self.key_for_ctx(ctx)).load()
+            self.s3.Object(self.bucket, tarball).load()
         except botocore.exceptions.ClientError as ex:
             if ex.response['Error']['Code'] == '404':
                 return False
@@ -33,18 +38,20 @@ class S3Backend(collections.namedtuple('S3Backend', (
             return True
 
     def get_artifact(self, ctx):
+        tarball, json = self.artifact_paths(ctx)
         fd, path = tempfile.mkstemp()
         os.close(fd)
         self.s3.Bucket(self.bucket).download_file(
-            self.key_for_ctx(ctx),
+            tarball,
             path,
         )
         return path
 
     def store_artifact(self, ctx, path):
+        tarball, json = self.artifact_paths(ctx)
         self.s3.Bucket(self.bucket).upload_file(
             path,
-            self.key_for_ctx(ctx),
+            tarball,
         )
 
     def invalidate_artifact(self, ctx):
diff --git a/lazy_build/context.py b/lazy_build/context.py
index eadebb9..d7e0009 100644
--- a/lazy_build/context.py
+++ b/lazy_build/context.py
@@ -96,7 +96,7 @@ def build_context(conf, command):
 def package_artifact(conf):
     fd, tmp = tempfile.mkstemp()
     os.close(fd)
-    with tarfile.TarFile(tmp, mode='w') as tf:
+    with tarfile.open(tmp, mode='w:gz') as tf:
         for output_path in conf.output:
             if os.path.isdir(output_path):
                 for path, _, filenames in os.walk(output_path):
@@ -115,5 +115,5 @@ def extract_artifact(conf, artifact):
             else:
                 os.remove(output_path)
 
-    with tarfile.TarFile(artifact) as tf:
+    with tarfile.open(artifact, 'r:gz') as tf:
         tf.extractall()


Test patch:
diff --git a/tests/context_test.py b/tests/context_test.py
index 1e58218..b3271f5 100644
--- a/tests/context_test.py
+++ b/tests/context_test.py
@@ -77,7 +77,7 @@ def test_package_artifact(tmpdir):
         after_download=None,
     ))
     try:
-        with tarfile.TarFile(tmp) as tf:
+        with tarfile.open(tmp, 'r:gz') as tf:
             members = {member.name for member in tf.getmembers()}
     finally:
         os.remove(tmp)
@@ -94,7 +94,7 @@ def test_extract_artifact(tmpdir):
     tmpdir.join('a/b/sup').ensure()
 
     tar = tmpdir.join('my.tar').strpath
-    with tarfile.TarFile(tar, 'w') as tf:
+    with tarfile.open(tar, 'w:gz') as tf:
         for path in ('my.txt', 'hello/there.txt', 'a/b/c/d.txt'):
             ti = tarfile.TarInfo(path)
             ti.size = 6