Repository: replicatedhq/kots
Pull request #69 (2019-09-24 18:19:30)

support new airgap bundle format


Patch:
diff --git a/pkg/image/builder.go b/pkg/image/builder.go
index f0ee557db..e7e603534 100644
--- a/pkg/image/builder.go
+++ b/pkg/image/builder.go
@@ -121,7 +121,8 @@ func saveOneImage(imagesDir string, image string) error {
 		return errors.Wrap(err, "failed to parse image ref")
 	}
 
-	pathInBundle := imageRef.pathInBundle()
+	imageFormat := "docker-archive"
+	pathInBundle := imageRef.pathInBundle(imageFormat)
 	archiveName := filepath.Join(imagesDir, pathInBundle)
 	destDir := filepath.Dir(archiveName)
 
@@ -143,9 +144,10 @@ func saveOneImage(imagesDir string, image string) error {
 		return errors.Wrap(err, "failed to parse source image name")
 	}
 
-	destRef, err := alltransports.ParseImageName(fmt.Sprintf("docker-archive:%s", archiveName))
+	destStr := fmt.Sprintf("%s:%s", imageFormat, archiveName)
+	destRef, err := alltransports.ParseImageName(destStr)
 	if err != nil {
-		return errors.Wrap(err, "failed to parse dest image name")
+		return errors.Wrapf(err, "failed to parse local image name: %s", destStr)
 	}
 
 	_, err = copy.Image(context.Background(), policyContext, destRef, srcRef, &copy.Options{
@@ -190,8 +192,8 @@ func imageRefImage(image string) (*ImageRef, error) {
 	return ref, nil
 }
 
-func (ref *ImageRef) pathInBundle() string {
-	path := []string{ref.Name}
+func (ref *ImageRef) pathInBundle(formatPrefix string) string {
+	path := []string{formatPrefix, ref.Name}
 	if ref.Tag != "" {
 		path = append(path, ref.Tag)
 	}
@@ -217,9 +219,10 @@ func CopyFromFileToRegistry(path string, name string, tag string, digest string)
 		return errors.Wrap(err, "failed to parse src image name")
 	}
 
-	destRef, err := alltransports.ParseImageName(fmt.Sprintf("docker://%s:%s", name, tag))
+	destStr := fmt.Sprintf("docker://%s:%s", name, tag)
+	destRef, err := alltransports.ParseImageName(destStr)
 	if err != nil {
-		return errors.Wrap(err, "failed to parse dest image name")
+		return errors.Wrapf(err, "failed to parse dest image name: %s", destStr)
 	}
 
 	_, err = copy.Image(context.Background(), policyContext, destRef, srcRef, &copy.Options{
diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go
index a0f4cc150..df8e0fd45 100644
--- a/pkg/pull/pull.go
+++ b/pkg/pull/pull.go
@@ -146,6 +146,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 		if pullOptions.RewriteImageOptions.Host != "" {
 			pushUpstreamImageOptions := upstream.PushUpstreamImageOptions{
 				RootDir:           pullOptions.RootDir,
+				ImagesDir:         imagesDirFromOptions(u, pullOptions),
 				CreateAppDir:      pullOptions.CreateAppDir,
 				Log:               log,
 				RegistryHost:      pullOptions.RewriteImageOptions.Host,
@@ -244,3 +245,15 @@ func parseLicenseFromFile(filename string) (*kotsv1beta1.License, error) {
 
 	return license.(*kotsv1beta1.License), nil
 }
+
+func imagesDirFromOptions(upstream *upstream.Upstream, pullOptions PullOptions) string {
+	if pullOptions.RewriteImageOptions.ImageFiles != "" {
+		return pullOptions.RewriteImageOptions.ImageFiles
+	}
+
+	if pullOptions.CreateAppDir {
+		return filepath.Join(pullOptions.RootDir, upstream.Name, "images")
+	}
+
+	return filepath.Join(pullOptions.RootDir, "images")
+}
diff --git a/pkg/upstream/push_images.go b/pkg/upstream/push_images.go
index 24dde45b8..c1ddabccf 100644
--- a/pkg/upstream/push_images.go
+++ b/pkg/upstream/push_images.go
@@ -1,6 +1,7 @@
 package upstream
 
 import (
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -14,6 +15,7 @@ import (
 
 type PushUpstreamImageOptions struct {
 	RootDir           string
+	ImagesDir         string
 	CreateAppDir      bool
 	Log               *logger.Logger
 	RegistryHost      string
@@ -25,42 +27,52 @@ func (u *Upstream) TagAndPushUpstreamImages(options PushUpstreamImageOptions) ([
 	if options.CreateAppDir {
 		rootDir = path.Join(rootDir, u.Name)
 	}
-	imagesDir := path.Join(rootDir, "images")
+
+	formatDirs, err := ioutil.ReadDir(options.ImagesDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read images dir")
+	}
 
 	images := []kustomizeimage.Image{}
+	for _, f := range formatDirs {
+		if !f.IsDir() {
+			continue
+		}
 
-	err := filepath.Walk(imagesDir,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
+		formatRoot := path.Join(options.ImagesDir, f.Name())
+		err := filepath.Walk(formatRoot,
+			func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
 
-			if info.IsDir() {
-				return nil
-			}
+				if info.IsDir() {
+					return nil
+				}
 
-			pathWithoutRoot := path[len(imagesDir)+1:]
+				pathWithoutRoot := path[len(formatRoot)+1:]
 
-			rewrittenImage, err := image.ImageInfoFromFile(options.RegistryHost, options.RegistryNamespace, strings.Split(pathWithoutRoot, string(os.PathSeparator)))
-			if err != nil {
-				return errors.Wrap(err, "failed to decode image from path")
-			}
+				rewrittenImage, err := image.ImageInfoFromFile(options.RegistryHost, options.RegistryNamespace, strings.Split(pathWithoutRoot, string(os.PathSeparator)))
+				if err != nil {
+					return errors.Wrap(err, "failed to decode image from path")
+				}
 
-			// copy to the registry
-			options.Log.ChildActionWithSpinner("Pushing image %s:%s", rewrittenImage.NewName, rewrittenImage.NewTag)
-			err = image.CopyFromFileToRegistry(path, rewrittenImage.NewName, rewrittenImage.NewTag, rewrittenImage.Digest)
-			if err != nil {
+				// copy to the registry
+				options.Log.ChildActionWithSpinner("Pushing image %s:%s", rewrittenImage.NewName, rewrittenImage.NewTag)
+				err = image.CopyFromFileToRegistry(path, rewrittenImage.NewName, rewrittenImage.NewTag, rewrittenImage.Digest)
+				if err != nil {
+					options.Log.FinishChildSpinner()
+					return errors.Wrap(err, "failed to push image")
+				}
 				options.Log.FinishChildSpinner()
-				return errors.Wrap(err, "failed to push image")
-			}
-			options.Log.FinishChildSpinner()
 
-			images = append(images, rewrittenImage)
-			return nil
-		})
+				images = append(images, rewrittenImage)
+				return nil
+			})
 
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to walk images dir")
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to walk images dir")
+		}
 	}
 
 	return images, nil


Test patch:
diff --git a/pkg/image/builder_test.go b/pkg/image/builder_test.go
index d112e2997..f6c9f8b46 100644
--- a/pkg/image/builder_test.go
+++ b/pkg/image/builder_test.go
@@ -18,8 +18,8 @@ func TestReleaseStartNum(t *testing.T) {
 		Tag:    "2.5",
 	}, ref)
 	require.Equal(t,
-		"docker.io/library/debian/2.5",
-		ref.pathInBundle())
+		"docker-archive/docker.io/library/debian/2.5",
+		ref.pathInBundle("docker-archive"))
 
 	ref, err = imageRefImage("debian")
 	require.NoError(t, err)
@@ -29,8 +29,8 @@ func TestReleaseStartNum(t *testing.T) {
 		Tag:    "latest",
 	}, ref)
 	require.Equal(t,
-		"docker.io/library/debian/latest",
-		ref.pathInBundle())
+		"oci-archive/docker.io/library/debian/latest",
+		ref.pathInBundle("oci-archive"))
 
 	ref, err = imageRefImage("quay.io/replicated/debian:2.5")
 	require.NoError(t, err)
@@ -40,8 +40,8 @@ func TestReleaseStartNum(t *testing.T) {
 		Tag:    "2.5",
 	}, ref)
 	require.Equal(t,
-		"quay.io/replicated/debian/2.5",
-		ref.pathInBundle())
+		"docker-archive/quay.io/replicated/debian/2.5",
+		ref.pathInBundle("docker-archive"))
 
 	ref, err = imageRefImage("myorg/ubuntu:14")
 	require.NoError(t, err)
@@ -51,8 +51,8 @@ func TestReleaseStartNum(t *testing.T) {
 		Tag:    "14",
 	}, ref)
 	require.Equal(t,
-		"docker.io/myorg/ubuntu/14",
-		ref.pathInBundle())
+		"docker-archive/docker.io/myorg/ubuntu/14",
+		ref.pathInBundle("docker-archive"))
 
 	ref, err = imageRefImage("myorg/ubuntu@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2")
 	require.NoError(t, err)
@@ -62,6 +62,6 @@ func TestReleaseStartNum(t *testing.T) {
 		Digest: "sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
 	}, ref)
 	require.Equal(t,
-		"docker.io/myorg/ubuntu/sha256/45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
-		ref.pathInBundle())
+		"docker-archive/docker.io/myorg/ubuntu/sha256/45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2",
+		ref.pathInBundle("docker-archive"))
 }