Repository: chainguard-dev/apko
Pull request #93 (2022-03-11 00:29:07)

feat: add support for chaining tarball context
This should help in melange

kaniini: Looks cool.  I think I see how we can use this to clean up the tarstreams in Melange!

Patch:
diff --git a/pkg/build/build.go b/pkg/build/build.go
index f6b0fdb..9c96700 100644
--- a/pkg/build/build.go
+++ b/pkg/build/build.go
@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"chainguard.dev/apko/pkg/build/types"
+	apkofs "chainguard.dev/apko/pkg/fs"
 	"chainguard.dev/apko/pkg/tarball"
 )
 
@@ -71,7 +72,7 @@ func (bc *Context) BuildTarball() (string, error) {
 		return "", fmt.Errorf("failed to construct tarball build context: %w", err)
 	}
 
-	if err := tw.WriteArchive(bc.WorkDir, outfile); err != nil {
+	if err := tw.WriteArchive(outfile, apkofs.DirFS(bc.WorkDir)); err != nil {
 		return "", fmt.Errorf("failed to generate tarball for image: %w", err)
 	}
 
diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go
new file mode 100644
index 0000000..51e0fca
--- /dev/null
+++ b/pkg/fs/fs.go
@@ -0,0 +1,38 @@
+// Copyright 2022 Chainguard, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type rlfs struct {
+	base string
+	f    fs.FS
+}
+
+func (f *rlfs) Readlink(name string) (string, error) {
+	return os.Readlink(filepath.Join(f.base, name))
+}
+
+func (f *rlfs) Open(name string) (fs.File, error) {
+	return f.f.Open(name)
+}
+
+func (f *rlfs) Stat(name string) (fs.FileInfo, error) {
+	return os.Stat(filepath.Join(f.base, name))
+}
diff --git a/pkg/fs/readlinkfs.go b/pkg/fs/readlinkfs.go
new file mode 100644
index 0000000..2e60fb2
--- /dev/null
+++ b/pkg/fs/readlinkfs.go
@@ -0,0 +1,33 @@
+// Copyright 2022 Chainguard, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fs
+
+import (
+	"io/fs"
+	"os"
+)
+
+type ReadLinkFS interface {
+	fs.FS
+
+	Readlink(name string) (string, error)
+}
+
+func DirFS(dir string) ReadLinkFS {
+	return &rlfs{
+		base: dir,
+		f:    os.DirFS(dir),
+	}
+}
diff --git a/pkg/tarball/append.go b/pkg/tarball/append.go
new file mode 100644
index 0000000..9ddd0c9
--- /dev/null
+++ b/pkg/tarball/append.go
@@ -0,0 +1,71 @@
+// Copyright 2022 Chainguard, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tarball
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"io/fs"
+)
+
+type MultiTar struct {
+	out *gzip.Writer
+}
+
+// Out returns a new MultiTar.
+func Out(dst io.Writer) *MultiTar {
+	return &MultiTar{
+		out: gzip.NewWriter(dst),
+	}
+}
+
+// Append appends to the underlying writer the content of the tarball
+// generated by the given context.
+// It also duplicates its writes to the extra writers, if any.
+func (m *MultiTar) Append(ctx *Context, src fs.FS, extra ...io.Writer) error {
+	all := make([]io.Writer, 0, len(extra)+1)
+	all = append(all, m.out)
+
+	for _, w := range extra {
+		all = append(all, gzip.NewWriter(w))
+	}
+
+	tw := tar.NewWriter(io.MultiWriter(all...))
+
+	if err := ctx.writeTar(tw, src); err != nil {
+		return err
+	}
+
+	tw.Flush()
+
+	if len(extra) != 0 {
+		// write tar and gzip footers to extra writers to make
+		// sure they get a valid archive.
+		for _, w := range all[1:] {
+			tar.NewWriter(w).Close()
+			w.(*gzip.Writer).Close()
+		}
+	}
+
+	return nil
+}
+
+// Close flushes and closes the underlying writer.
+func (m *MultiTar) Close() {
+	// write tar and gzip footers to the main writer.
+	tar.NewWriter(m.out).Close()
+	m.out.Close()
+}
diff --git a/pkg/tarball/tarball.go b/pkg/tarball/tarball.go
index e0a106e..3e0ddf5 100644
--- a/pkg/tarball/tarball.go
+++ b/pkg/tarball/tarball.go
@@ -15,15 +15,6 @@
 package tarball
 
 import (
-	"archive/tar"
-	"compress/gzip"
-	"crypto/sha1" // nolint:gosec
-	"encoding/hex"
-	"fmt"
-	"io"
-	"io/fs"
-	"os"
-	"path/filepath"
 	"time"
 )
 
@@ -105,123 +96,3 @@ func WithUseChecksums(useChecksums bool) Option {
 		return nil
 	}
 }
-
-// Writes a raw TAR archive to out, given an fs.FS.
-func (ctx *Context) WriteArchiveFromFS(base string, fsys fs.FS, out io.Writer) error {
-	gzw := gzip.NewWriter(out)
-	defer gzw.Close()
-
-	tw := tar.NewWriter(gzw)
-	if !ctx.SkipClose {
-		defer tw.Close()
-	} else {
-		defer tw.Flush()
-	}
-
-	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
-		// skip the root path, superfluous
-		if path == "." {
-			return nil
-		}
-
-		if err != nil {
-			return err
-		}
-
-		info, err := d.Info()
-		if err != nil {
-			return err
-		}
-
-		var link string
-		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
-			// fs.FS does not implement readlink, so we have this hack for now.
-			if link, err = os.Readlink(filepath.Join(base, path)); err != nil {
-				return err
-			}
-		}
-
-		header, err := tar.FileInfoHeader(info, link)
-		if err != nil {
-			return err
-		}
-		// work around some weirdness, without this we wind up with just the basename
-		header.Name = path
-
-		// zero out timestamps for reproducibility
-		header.AccessTime = ctx.SourceDateEpoch
-		header.ModTime = ctx.SourceDateEpoch
-		header.ChangeTime = ctx.SourceDateEpoch
-
-		if ctx.OverrideUIDGID {
-			header.Uid = ctx.UID
-			header.Gid = ctx.GID
-		}
-
-		if ctx.OverrideUname != "" {
-			header.Uname = ctx.OverrideUname
-		}
-
-		if ctx.OverrideGname != "" {
-			header.Gname = ctx.OverrideGname
-		}
-
-		if ctx.UseChecksums {
-			header.PAXRecords = map[string]string{}
-
-			if link != "" {
-				linkDigest := sha1.Sum([]byte(link)) // nolint:gosec
-				linkChecksum := hex.EncodeToString(linkDigest[:])
-				header.PAXRecords["APK-TOOLS.checksum.SHA1"] = linkChecksum
-			} else if info.Mode().IsRegular() {
-				data, err := fsys.Open(path)
-				if err != nil {
-					return err
-				}
-				defer data.Close()
-
-				fileDigest := sha1.New() // nolint:gosec
-				if _, err := io.Copy(fileDigest, data); err != nil {
-					return err
-				}
-
-				fileChecksum := hex.EncodeToString(fileDigest.Sum(nil))
-				header.PAXRecords["APK-TOOLS.checksum.SHA1"] = fileChecksum
-			}
-		}
-
-		if err := tw.WriteHeader(header); err != nil {
-			return err
-		}
-
-		if info.Mode().IsRegular() {
-			data, err := fsys.Open(path)
-			if err != nil {
-				return err
-			}
-
-			defer data.Close()
-
-			if _, err := io.Copy(tw, data); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Writes a tarball to a temporary file.  Caller's responsibility to
-// clean it up when it's done with it.
-func (ctx *Context) WriteArchive(src string, w io.Writer) error {
-	fs := os.DirFS(src)
-	if err := ctx.WriteArchiveFromFS(src, fs, w); err != nil {
-		return fmt.Errorf("writing TAR archive failed: %w", err)
-	}
-
-	return nil
-}
diff --git a/pkg/tarball/write.go b/pkg/tarball/write.go
new file mode 100644
index 0000000..45f84b6
--- /dev/null
+++ b/pkg/tarball/write.go
@@ -0,0 +1,149 @@
+// Copyright 2022 Chainguard, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tarball
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"crypto/sha1" // nolint:gosec
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+
+	apkofs "chainguard.dev/apko/pkg/fs"
+)
+
+func (ctx *Context) writeArchiveFromFS(dst io.Writer, fsys fs.FS) error {
+	gzw := gzip.NewWriter(dst)
+	defer gzw.Close()
+
+	tw := tar.NewWriter(gzw)
+	defer tw.Close()
+
+	return ctx.writeTar(tw, fsys)
+}
+
+func (ctx *Context) writeTar(tw *tar.Writer, fsys fs.FS) error {
+	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		// skip the root path, superfluous
+		if path == "." {
+			return nil
+		}
+
+		if err != nil {
+			return err
+		}
+
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		var link string
+		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
+			rlfs, ok := fsys.(apkofs.ReadLinkFS)
+			if !ok {
+				return fmt.Errorf("readlink not supported by this fs: path (%s)", path)
+			}
+
+			if link, err = rlfs.Readlink(path); err != nil {
+				return err
+			}
+		}
+
+		header, err := tar.FileInfoHeader(info, link)
+		if err != nil {
+			return err
+		}
+		// work around some weirdness, without this we wind up with just the basename
+		header.Name = path
+
+		// zero out timestamps for reproducibility
+		header.AccessTime = ctx.SourceDateEpoch
+		header.ModTime = ctx.SourceDateEpoch
+		header.ChangeTime = ctx.SourceDateEpoch
+
+		if ctx.OverrideUIDGID {
+			header.Uid = ctx.UID
+			header.Gid = ctx.GID
+		}
+
+		if ctx.OverrideUname != "" {
+			header.Uname = ctx.OverrideUname
+		}
+
+		if ctx.OverrideGname != "" {
+			header.Gname = ctx.OverrideGname
+		}
+
+		if ctx.UseChecksums {
+			header.PAXRecords = map[string]string{}
+
+			if link != "" {
+				linkDigest := sha1.Sum([]byte(link)) // nolint:gosec
+				linkChecksum := hex.EncodeToString(linkDigest[:])
+				header.PAXRecords["APK-TOOLS.checksum.SHA1"] = linkChecksum
+			} else if info.Mode().IsRegular() {
+				data, err := fsys.Open(path)
+				if err != nil {
+					return err
+				}
+				defer data.Close()
+
+				fileDigest := sha1.New() // nolint:gosec
+				if _, err := io.Copy(fileDigest, data); err != nil {
+					return err
+				}
+
+				fileChecksum := hex.EncodeToString(fileDigest.Sum(nil))
+				header.PAXRecords["APK-TOOLS.checksum.SHA1"] = fileChecksum
+			}
+		}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		if info.Mode().IsRegular() {
+			data, err := fsys.Open(path)
+			if err != nil {
+				return err
+			}
+
+			defer data.Close()
+
+			if _, err := io.Copy(tw, data); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// WriteArchive writes a tarball to the provided io.Writer.
+func (ctx *Context) WriteArchive(dst io.Writer, src fs.FS) error {
+	if err := ctx.writeArchiveFromFS(dst, src); err != nil {
+		return fmt.Errorf("writing TAR archive failed: %w", err)
+	}
+
+	return nil
+}


Test patch:
diff --git a/pkg/tarball/append_test.go b/pkg/tarball/append_test.go
new file mode 100644
index 0000000..f869ece
--- /dev/null
+++ b/pkg/tarball/append_test.go
@@ -0,0 +1,71 @@
+// Copyright 2022 Chainguard, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tarball_test
+
+import (
+	"bytes"
+	"embed"
+	"io"
+	"testing"
+
+	"chainguard.dev/apko/pkg/tarball"
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed testdata
+var testdataFS embed.FS
+
+//go:embed testdata/foo
+var fooFS embed.FS
+
+//go:embed testdata/bar
+var barFS embed.FS
+
+func TestMultiTar(t *testing.T) {
+	ctx, err := tarball.NewContext()
+	require.NoError(t, err)
+
+	var got bytes.Buffer
+
+	m := tarball.Out(&got)
+	require.NoError(t, m.Append(ctx, testdataFS))
+	m.Close()
+
+	var expected bytes.Buffer
+	require.NoError(t, ctx.WriteArchive(&expected, testdataFS))
+
+	require.Equal(t, expected, got)
+}
+
+func TestExtraWriter(t *testing.T) {
+	ctx, err := tarball.NewContext()
+	require.NoError(t, err)
+
+	var gotFoo bytes.Buffer
+	var gotBar bytes.Buffer
+
+	m := tarball.Out(io.Discard)
+	require.NoError(t, m.Append(ctx, barFS, &gotBar))
+	require.NoError(t, m.Append(ctx, fooFS, &gotFoo))
+	m.Close()
+
+	var expectedFoo bytes.Buffer
+	var expectedBar bytes.Buffer
+	require.NoError(t, ctx.WriteArchive(&expectedBar, barFS))
+	require.NoError(t, ctx.WriteArchive(&expectedFoo, fooFS))
+
+	require.Equal(t, expectedFoo, gotFoo)
+	require.Equal(t, expectedBar, gotBar)
+}
diff --git a/pkg/tarball/testdata/bar/bar.txt b/pkg/tarball/testdata/bar/bar.txt
new file mode 100644
index 0000000..5716ca5
--- /dev/null
+++ b/pkg/tarball/testdata/bar/bar.txt
@@ -0,0 +1,1 @@
+bar
diff --git a/pkg/tarball/testdata/foo/foo.txt b/pkg/tarball/testdata/foo/foo.txt
new file mode 100644
index 0000000..257cc56
--- /dev/null
+++ b/pkg/tarball/testdata/foo/foo.txt
@@ -0,0 +1,1 @@
+foo