Repository: tetratelabs/getenvoy (go)

Issue:
Adjust for docker.io rate limits
**Describe the bug**
Builds can fail due to docker.io rate limits like so:
```
Unable to find image 'busybox:latest' locally
docker: Error response from daemon: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit.
See 'docker run --help'.
```

**To Reproduce**
do a bunch of builds, then look at the logs

**Expected behavior**
builds run

**Additional context**
This is caused by using base images hosted by docker.io we can use a different registry for base images to avoid this.


Pull request:
Fixes docker.io normal case rate-limiting
Each pull request implies many docker.io requests, this has led to
builds failing on account of rate limiting. This avoids that by using
the GitHub Container Registry (ghcr.io) instead of Docker Hub
(docker.io). ghcr.io is not rate-limited. Specifically, we use internal
copies of images used as parents or in testing. Since these aren't what
end users use anyway, this doesn't impact them.

At the end of the day, ghcr.io/tetratelabs/getenvoy-internal has tags
corresponding to our needs:

```yaml
          - source: busybox:1.32.1  # test image: ci/e2e/darwin/install_docker.sh
            target_tag: busybox
          - source: registry:2  # test image: docker-compose.yml
            target_tag: registry
          - source: rust:1.51.0  # parent image: images/extension-builders/rust/Dockerfile
            target_tag: rust
          - source: tinygo/tinygo:0.17.0  # parent image: images/extension-builders/tinygo/Dockerfile
            target_tag: tinygo
```

Updating our base layers or otherwise implies a change to
`.github/workflows/internal-images.yml` and re-triggering the workflow
or waiting until the next day for automatic publishing to occur.

Another change in this PR is removing the docker dependency from unit
tests via a mock. Even if we have safe images, we still shouldn't
subject unit tests to flakiness, extra time, or network dependencies
unless there is no other way.

Fixes #149

Signed-off-by: Adrian Cole <adrian@tetrate.io>

Patch:
diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml
index 64d90d4..1b20b25 100644
--- a/.github/workflows/commit.yaml
+++ b/.github/workflows/commit.yaml
@@ -35,13 +35,6 @@ jobs:
         with:
           go-version: '1.16.2'
 
-      - name: "Install 'Docker for Mac' (Latest)"
-        uses: docker-practice/actions-setup-docker@v1  # needed while TestGetEnvoyExtensionPush uses a real registry
-        if: runner.os == 'macOS'
-        with:
-          docker_buildx: false  # Install is flakey. When it, we can install it via docker/setup-buildx-action@v1
-        timeout-minutes: 20  # fail fast if MacOS install takes too long
-
       - name: "Verify clean check-in"
         run: make check
 
@@ -138,7 +131,7 @@ jobs:
 
       - name: "Build language-specific Docker build images"
         run: make builders
-        timeout-minutes: 10  # fail fast if MacOS runner becomes too slow
+        timeout-minutes: 20  # NOTE: the rust image is very large and can alone take 7 minutes to download and build
 
       - name: "Run e2e tests using the `getenvoy` binary built by the upstream job"
         # chmod to restore permissions lost in actions/download-artifact@v2
diff --git a/.github/workflows/internal-images.yml b/.github/workflows/internal-images.yml
new file mode 100644
index 0000000..f06aa0f
--- /dev/null
+++ b/.github/workflows/internal-images.yml
@@ -0,0 +1,46 @@
+# yamllint --format github .github/workflows/internal-images.yml
+---
+name: internal-images
+
+# Refresh the tags once a day
+on:
+  schedule:
+    - cron: "23 3 * * *"
+  workflow_dispatch:  # Allows manual refresh
+
+# This copies images from docker.io to ghcr.io/tetratelabs/getenvoy-internal:$tag
+# Using these in tests and as a parent (FROM) avoids docker.io rate-limits particularly on pull requests.
+jobs:
+  copy-images:
+    strategy:
+      matrix:
+        # Be precise in tag versions to improve reproducibility
+        include:
+          - source: busybox:1.32.1  # test image: ci/e2e/darwin/install_docker.sh
+            target_tag: busybox
+          - source: registry:2  # test image: docker-compose.yml
+            target_tag: registry
+          - source: rust:1.51.0  # parent image: images/extension-builders/rust/Dockerfile
+            target_tag: rust
+          - source: tinygo/tinygo:0.17.0  # parent image: images/extension-builders/tinygo/Dockerfile
+            target_tag: tinygo
+    runs-on: ubuntu-latest
+    steps:
+      # Same as doing this locally: echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_TOKEN}" --password-stdin
+      - name: Login into GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          # GHCR_TOKEN=<hex token value>
+          #   - pushes Docker images to ghcr.io
+          #   - create via https://github.com/settings/tokens
+          #   - assign via https://github.com/organizations/tetratelabs/settings/secrets/actions
+          #   - needs repo:status, public_repo, write:packages, delete:packages
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Pull and push
+        run: |  # This will only push a single architecture, which is fine as we currently only support amd64
+          docker pull ${{ matrix.source }}
+          docker tag ${{ matrix.source }} ghcr.io/tetratelabs/getenvoy-internal:${{ matrix.target_tag }}
+          docker push ghcr.io/tetratelabs/getenvoy-internal:${{ matrix.target_tag }}
diff --git a/Makefile b/Makefile
index 2513a3b..da781fe 100644
--- a/Makefile
+++ b/Makefile
@@ -88,7 +88,6 @@ release.dryrun:
 
 .PHONY: test
 test:
-	docker-compose up -d
 	go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)
 
 .PHONY: e2e
diff --git a/docker-compose.yml b/docker-compose.yml
index d8a6f27..4f135cc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,6 +15,8 @@
 version: '3'
 services:
   registry:
-    image: registry:2
+    # ghcr.io allows end-to-end (e2e) tests to avoid docker.io rate limits
+    # .github/workflows/internal-images.yml publishes this
+    image: ghcr.io/tetratelabs/getenvoy-internal:registry
     ports:
       - "5000:5000"
diff --git a/images/extension-builders/rust/Dockerfile b/images/extension-builders/rust/Dockerfile
index 0ac4d13..981346b 100644
--- a/images/extension-builders/rust/Dockerfile
+++ b/images/extension-builders/rust/Dockerfile
@@ -15,7 +15,9 @@
 #
 # Builder image for Envoy Wasm extensions written in Rust.
 #
-FROM rust:1.44.1
+# ghcr.io allows building without using docker.io as it is rate-limited
+# .github/workflows/internal-images.yml publishes this
+FROM ghcr.io/tetratelabs/getenvoy-internal:rust
 
 RUN rustup target add wasm32-unknown-unknown
 
diff --git a/images/extension-builders/tinygo/Dockerfile b/images/extension-builders/tinygo/Dockerfile
index c9c7611..ba5fd86 100644
--- a/images/extension-builders/tinygo/Dockerfile
+++ b/images/extension-builders/tinygo/Dockerfile
@@ -15,8 +15,9 @@
 #
 # Builder image for Envoy Wasm extensions written in Go.
 #
-
-FROM tinygo/tinygo:0.17.0
+# ghcr.io allows building without using docker.io as it is rate-limited
+# .github/workflows/internal-images.yml publishes this
+FROM ghcr.io/tetratelabs/getenvoy-internal:tinygo
 
 ENV GOCACHE=/source/build/.gocache
 ENV GOMODCACHE=/source/build/.gomodcache


Test patch:
diff --git a/ci/e2e/darwin/install_docker.sh b/ci/e2e/darwin/install_docker.sh
index e6b4381..f2c7f6c 100755
--- a/ci/e2e/darwin/install_docker.sh
+++ b/ci/e2e/darwin/install_docker.sh
@@ -38,5 +38,6 @@ while ! docker info 2> /dev/null; do
 	If Docker is not ready within 10 minutes, it's better to fail the current CI Job and let the comitter to re-start the job manually"
 done
 
-# sanity check without using docker.io as it is rate-limited
-docker run --rm quay.io/prometheus/busybox date
+# Verify install without using docker.io as it is rate-limited
+# .github/workflows/internal-images.yml publishes this
+docker run --rm ghcr.io/tetratelabs/getenvoy-internal:busybox date
diff --git a/pkg/cmd/extension/push/cmd_test.go b/pkg/cmd/extension/push/cmd_test.go
index 12c12c6..31173ec 100644
--- a/pkg/cmd/extension/push/cmd_test.go
+++ b/pkg/cmd/extension/push/cmd_test.go
@@ -15,8 +15,14 @@
 package push_test
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -29,15 +35,17 @@ import (
 // relativeWorkspaceDir points to a usable pre-initialized workspace
 const relativeWorkspaceDir = "testdata/workspace"
 
-// localRegistryWasmImageRef corresponds to a Docker container running the image "registry:2"
-// As this is not intended to be an end-to-end test, this could be improved to use a mock/fake HTTP registry instead.
-const localRegistryWasmImageRef = "localhost:5000/getenvoy/sample"
-
 // When unspecified, we default the tag to Docker's default "latest". Note: recent tools enforce qualifying this!
 const defaultTag = "latest"
 
 // TestGetEnvoyExtensionPush shows current directory is usable, provided it is a valid workspace.
 func TestGetEnvoyExtensionPush(t *testing.T) {
+	mock := mockRegistryServer(t)
+	defer mock.Close()
+
+	// localhost:5000/getenvoy/sample, not http://localhost:5000/getenvoy/sample
+	localRegistryWasmImageRef := fmt.Sprintf(`%s/getenvoy/sample`, mock.Listener.Addr())
+
 	_, revertWd := RequireChDir(t, relativeWorkspaceDir)
 	defer revertWd()
 
@@ -51,7 +59,6 @@ func TestGetEnvoyExtensionPush(t *testing.T) {
 
 	// Verify stdout shows the latest tag and the correct image ref
 	require.NoError(t, err, `expected no error running [%v]`, c)
-
 	require.Contains(t, stdout.String(), fmt.Sprintf(`Using default tag: %s
 Pushed %s
 digest: sha256`, defaultTag, imageRef), `unexpected stderr after running [%v]`, c)
@@ -59,6 +66,12 @@ digest: sha256`, defaultTag, imageRef), `unexpected stderr after running [%v]`,
 }
 
 func TestGetEnvoyExtensionPushFailsOutsideWorkspaceDirectory(t *testing.T) {
+	mock := mockRegistryServer(t)
+	defer mock.Close()
+
+	// localhost:5000/getenvoy/sample, not http://localhost:5000/getenvoy/sample
+	localRegistryWasmImageRef := fmt.Sprintf(`%s/getenvoy/sample`, mock.Listener.Addr())
+
 	// Change to a non-workspace dir
 	dir, revertWd := RequireChDir(t, relativeWorkspaceDir+"/..")
 	defer revertWd()
@@ -78,6 +91,12 @@ func TestGetEnvoyExtensionPushFailsOutsideWorkspaceDirectory(t *testing.T) {
 
 // TestGetEnvoyExtensionPushWithExplicitFileOption shows we don't need to be in a workspace directory to push a wasm.
 func TestGetEnvoyExtensionPushWithExplicitFileOption(t *testing.T) {
+	mock := mockRegistryServer(t)
+	defer mock.Close()
+
+	// localhost:5000/getenvoy/sample, not http://localhost:5000/getenvoy/sample
+	localRegistryWasmImageRef := fmt.Sprintf(`%s/getenvoy/sample`, mock.Listener.Addr())
+
 	// Change to a non-workspace dir
 	dir, revertWd := RequireChDir(t, relativeWorkspaceDir+"/..")
 	defer revertWd()
@@ -97,3 +116,46 @@ Pushed %s:latest
 digest: sha256`, localRegistryWasmImageRef))
 	require.Empty(t, stderr, `expected no stderr running [%v]`, c)
 }
+
+// The tests above are unit tests, not end-to-end (e2e) tests. Hence, we use a mock registry instead of a real one.
+func mockRegistryServer(t *testing.T) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		statusCode := 500
+
+		body, err := ioutil.ReadAll(r.Body) // fully read the request
+		require.NoError(t, err, "Error reading body of %s %s", r.Method, r.URL.Path)
+
+		switch r.Method {
+		case "HEAD":
+			if strings.Index(r.URL.Path, "/v2/getenvoy/sample/blobs") == 0 {
+				statusCode = 404 // pretend it hasn't been uploaded, yet
+			} else if r.URL.Path == "/v2/getenvoy/sample/manifests/latest" {
+				statusCode = 404 // pretend there's no manifest either
+			}
+		case "POST":
+			if r.URL.Path == "/v2/getenvoy/sample/blobs/uploads/" {
+				statusCode = 202 // pretend we processed the data
+				w.Header().Add("Location", "/upload")
+			}
+		case "PUT":
+			if r.URL.Path == "/upload" {
+				err := r.ParseForm()
+				require.NoError(t, err, "Error parsing PUT %s", r.URL.Path)
+				require.NotEmpty(t, r.Form.Get("digest"), `Expected PUT %s to have a query parameter "digest"`, r.URL.Path)
+
+				w.Header().Add("Docker-Content-Digest", r.Form.Get("digest"))
+				statusCode = 200 // Pretend we accepted the blob
+			} else if r.URL.Path == "/v2/getenvoy/sample/manifests/latest" {
+				w.Header().Add("Docker-Content-Digest", "sha256:"+hash(body))
+				statusCode = 200 // Pretend we accepted the manifest
+			}
+		}
+		w.WriteHeader(statusCode)
+	}))
+}
+
+func hash(b []byte) string {
+	h := sha256.New()
+	h.Write(b)
+	return hex.EncodeToString(h.Sum(nil))
+}