Repository: googleforgames/open-match
Pull request #503 (2019-06-07 18:13:47)

Create cluster reaper for Open Match e2e tests. (#88)
This is a command line tool to purge leaked clusters that will occur when there's CI test failures during cluster based end to end tests.

This command line tool may not exist as is when we need to actually start doing e2e testing. It might change to a Cloud Scheduler job.

jeremyje: /gcbrun

Patch:
diff --git a/.dockerignore b/.dockerignore
index e4102ef9..18beae58 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -111,6 +111,7 @@ cmd/swaggerui/swaggerui
 tools/certgen/certgen
 examples/demo/demo
 examples/functions/golang/soloduel/soloduel
+tools/reaper/reaper
 
 # Open Match Build Directory
 build/
diff --git a/.gitignore b/.gitignore
index 825a4785..91ad23ec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -111,6 +111,7 @@ cmd/swaggerui/swaggerui
 tools/certgen/certgen
 examples/demo/demo
 examples/functions/golang/soloduel/soloduel
+tools/reaper/reaper
 
 # Secrets Directories
 install/helm/open-match/secrets/
diff --git a/Makefile b/Makefile
index de2b6e36..e0a14629 100644
--- a/Makefile
+++ b/Makefile
@@ -135,6 +135,9 @@ DASHBOARD_PORT = 9092
 # AppEngine variables
 GAE_SITE_VERSION = om$(YEAR_MONTH)
 
+# Open Match Cluster E2E Test Variables
+OPEN_MATCH_CI_LABEL = open-match-ci
+
 # If the version is 0.0* then the service name is "development" as in development.open-match.dev.
 ifeq ($(MAJOR_MINOR_VERSION),0.0)
 	GAE_SERVICE_NAME = development
@@ -447,7 +450,7 @@ set-redis-password:
 install-toolchain: install-kubernetes-tools install-web-tools install-protoc-tools install-openmatch-tools
 install-kubernetes-tools: build/toolchain/bin/kubectl$(EXE_EXTENSION) build/toolchain/bin/helm$(EXE_EXTENSION) build/toolchain/bin/minikube$(EXE_EXTENSION) build/toolchain/bin/skaffold$(EXE_EXTENSION)
 install-protoc-tools: build/toolchain/bin/protoc$(EXE_EXTENSION) build/toolchain/bin/protoc-gen-go$(EXE_EXTENSION) build/toolchain/bin/protoc-gen-grpc-gateway$(EXE_EXTENSION) build/toolchain/bin/protoc-gen-swagger$(EXE_EXTENSION)
-install-openmatch-tools: build/toolchain/bin/certgen$(EXE_EXTENSION)
+install-openmatch-tools: build/toolchain/bin/certgen$(EXE_EXTENSION) build/toolchain/bin/reaper$(EXE_EXTENSION)
 
 build/toolchain/bin/helm$(EXE_EXTENSION):
 	mkdir -p $(TOOLCHAIN_BIN)
@@ -519,6 +522,10 @@ build/toolchain/bin/certgen$(EXE_EXTENSION): tools/certgen/certgen$(EXE_EXTENSIO
 	mkdir -p $(TOOLCHAIN_BIN)
 	cp -f $(REPOSITORY_ROOT)/tools/certgen/certgen$(EXE_EXTENSION) $(CERTGEN)
 
+build/toolchain/bin/reaper$(EXE_EXTENSION): tools/reaper/reaper$(EXE_EXTENSION)
+	mkdir -p $(TOOLCHAIN_BIN)
+	cp -f $(REPOSITORY_ROOT)/tools/reaper/reaper$(EXE_EXTENSION) $(TOOLCHAIN_BIN)/reaper$(EXE_EXTENSION)
+
 push-helm: build/toolchain/bin/helm$(EXE_EXTENSION) build/toolchain/bin/kubectl$(EXE_EXTENSION)
 	$(KUBECTL) create serviceaccount --namespace kube-system tiller
 	$(HELM) init --service-account tiller --force-upgrade
@@ -699,6 +706,9 @@ cmd/swaggerui/swaggerui$(EXE_EXTENSION): site/static/swaggerui/
 tools/certgen/certgen$(EXE_EXTENSION):
 	cd $(REPOSITORY_ROOT)/tools/certgen/ && $(GO_BUILD_COMMAND)
 
+tools/reaper/reaper$(EXE_EXTENSION):
+	cd tools/reaper/ && $(GO_BUILD_COMMAND)
+
 build/policies/binauthz.yaml: install/policies/binauthz.yaml
 	mkdir -p $(BUILD_DIR)/policies
 	cp -f $(REPOSITORY_ROOT)/install/policies/binauthz.yaml $(BUILD_DIR)/policies/binauthz.yaml
@@ -728,6 +738,9 @@ else
 	@echo "Not deploying build artifacts to open-match.dev because this is not a post commit change."
 endif
 
+ci-reap-clusters: build/toolchain/bin/reaper$(EXE_EXTENSION)
+	$(TOOLCHAIN_BIN)/reaper -project=$(GCP_PROJECT_ID) -age=30m -location=$(GCP_ZONE) -label=$(OPEN_MATCH_CI_LABEL)
+
 # For presubmit we want to update the protobuf generated files and verify that tests are good.
 presubmit: update-deps third_party clean-protos clean-secrets all-protos lint build test clean-site site-test md-test
 
@@ -749,13 +762,15 @@ clean-protos:
 	rm -rf $(REPOSITORY_ROOT)/internal/pb/
 
 clean-binaries:
-	rm -rf $(REPOSITORY_ROOT)/cmd/backend/backend
-	rm -rf $(REPOSITORY_ROOT)/cmd/evaluator/evaluator
-	rm -rf $(REPOSITORY_ROOT)/cmd/frontend/frontend
-	rm -rf $(REPOSITORY_ROOT)/cmd/mmlogic/mmlogic
-	rm -rf $(REPOSITORY_ROOT)/cmd/minimatch/minimatch
-	rm -rf $(REPOSITORY_ROOT)/examples/functions/golang/soloduel/soloduel
-	rm -rf $(REPOSITORY_ROOT)/cmd/swaggerui/swaggerui
+	rm -rf $(REPOSITORY_ROOT)/cmd/backend/backend$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/cmd/evaluator/evaluator$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/cmd/frontend/frontend$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/cmd/mmlogic/mmlogic$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/cmd/minimatch/minimatch$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/examples/functions/golang/soloduel/soloduel$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/cmd/swaggerui/swaggerui$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/tools/certgen/certgen$(EXE_EXTENSION)
+	rm -rf $(REPOSITORY_ROOT)/tools/reaper/reaper$(EXE_EXTENSION)
 
 clean-build: clean-toolchain clean-archives clean-release
 	rm -rf $(BUILD_DIR)/
@@ -873,7 +888,7 @@ ifeq ($(shell whoami),root)
 endif
 endif
 
-.PHONY: docker gcloud deploy-redirect-site update-deps sync-deps sleep-10 proxy-dashboard proxy-prometheus proxy-grafana clean clean-build clean-toolchain clean-archives clean-binaries clean-protos presubmit test ci-test site-test md-test vet
+.PHONY: docker gcloud deploy-redirect-site update-deps sync-deps sleep-10 proxy-dashboard proxy-prometheus proxy-grafana clean clean-build clean-toolchain clean-archives clean-binaries clean-protos presubmit test ci-reap-clusters ci-test site-test md-test vet
 
 
 ##########################################################################
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 1c523b0d..813175b2 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -144,6 +144,11 @@ steps:
   - name: 'go-vol'
     path: '/go'
 
+- id: 'Test: Delete Old Clusters'
+  name: 'gcr.io/$PROJECT_ID/open-match-build'
+  args: ['make', 'ci-reap-clusters']
+  waitFor: ['Build: Install Toolchain']
+
 artifacts:
     objects:
         location: gs://open-match-build-artifacts/output/
diff --git a/go.mod b/go.mod
index feab2d53..bf4ad607 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ module open-match.dev/open-match
 go 1.12
 
 require (
+	cloud.google.com/go v0.37.4
 	contrib.go.opencensus.io/exporter/jaeger v0.1.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	contrib.go.opencensus.io/exporter/stackdriver v0.11.0
diff --git a/tools/reaper/internal/internal.go b/tools/reaper/internal/internal.go
new file mode 100644
index 00000000..a687b162
--- /dev/null
+++ b/tools/reaper/internal/internal.go
@@ -0,0 +1,116 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package internal holds the internal workings for deleting orphaned GKE clusters from Continuous Integration.
+package internal
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	container "cloud.google.com/go/container/apiv1"
+	containerpb "google.golang.org/genproto/googleapis/container/v1"
+)
+
+// Params for deleting a cluster.
+type Params struct {
+	Age       time.Duration
+	Label     string
+	ProjectID string
+	Location  string
+}
+
+// ReapCluster scans for orphaned clusters and deletes them.
+func ReapCluster(params *Params) {
+	log.Printf("Scanning for orphaned clusters in projects/%s/locations/%s that are older than %s with label %s.", params.ProjectID, params.Location, params.Age.String(), params.Label)
+	ctx := context.Background()
+	c, err := container.NewClusterManagerClient(ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+	clusters, err := listOrphanedClusters(ctx, c, params)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = deleteClusters(ctx, c, clusters)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(clusters) > 0 {
+		log.Printf("Deleted %d GKE clusters", len(clusters))
+	} else {
+		log.Printf("There were no orphaned clusters. :)")
+	}
+}
+
+func listOrphanedClusters(ctx context.Context, c *container.ClusterManagerClient, params *Params) ([]*containerpb.Cluster, error) {
+	resp, err := c.ListClusters(ctx, &containerpb.ListClustersRequest{
+		Parent: fmt.Sprintf("projects/%s/locations/%s", params.ProjectID, params.Location),
+	})
+	if err != nil {
+		return []*containerpb.Cluster{}, err
+	}
+	orphanedClusters := []*containerpb.Cluster{}
+	for _, cluster := range resp.Clusters {
+		if isOrphaned(cluster, params) {
+			log.Printf("Cluster '/%s' was orphaned.\nDetails: %+v", cluster.Name, cluster)
+			orphanedClusters = append(orphanedClusters, cluster)
+		}
+	}
+	return orphanedClusters, nil
+}
+
+func deleteClusters(ctx context.Context, c *container.ClusterManagerClient, clusters []*containerpb.Cluster) error {
+	for _, cluster := range clusters {
+		clusterName := selfLinkToQualifiedName(cluster.SelfLink)
+		log.Printf("Deleting Orphaned GKE Cluster %s", clusterName)
+		resp, err := c.DeleteCluster(ctx, &containerpb.DeleteClusterRequest{
+			Name: clusterName,
+		})
+		if err != nil {
+			return err
+		}
+		log.Printf("Delete GKE Cluster: %s => %v", clusterName, resp)
+	}
+	return nil
+}
+
+func isOrphaned(cluster *containerpb.Cluster, params *Params) bool {
+	deadline := time.Now().Add(-1 * params.Age)
+	creationTime, err := time.Parse(time.RFC3339, cluster.CreateTime)
+	if err != nil {
+		log.Printf("cannot parse time '%s' because '%v', ignoring for orphan detection.", cluster.CreateTime, err)
+		return false
+	}
+	if creationTime.After(deadline) {
+		return false
+	}
+	if _, ok := cluster.ResourceLabels[params.Label]; !ok {
+		return false
+	}
+	return true
+}
+
+func selfLinkToQualifiedName(selfLink string) string {
+	parts := strings.Split(selfLink, "v1/")
+	if len(parts) != 2 {
+		log.Fatalf("SelfLink: %s is not a valid selflink it should contain 'v1/'", selfLink)
+	}
+	return parts[1]
+}
diff --git a/tools/reaper/reaper.go b/tools/reaper/reaper.go
new file mode 100644
index 00000000..ffac0d61
--- /dev/null
+++ b/tools/reaper/reaper.go
@@ -0,0 +1,40 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main is the main for the reaper tool to cleanup leaked clusters in Open Match CI.
+package main
+
+import (
+	"flag"
+	"time"
+
+	reaperInternal "open-match.dev/open-match/tools/reaper/internal"
+)
+
+var (
+	projectIDFlag = flag.String("project", "open-match-build", "Target Project ID to scan for leaked clusters.")
+	ageFlag       = flag.Duration("age", time.Hour*1, "Age of a cluster before it's a candidate for deletion.")
+	labelFlag     = flag.String("label", "open-match-ci", "Required label that must be on the cluster for it to be considered for reaping.")
+	locationFlag  = flag.String("location", "us-west1-a", "Location of the cluster")
+)
+
+func main() {
+	flag.Parse()
+	reaperInternal.ReapCluster(&reaperInternal.Params{
+		Age:       *ageFlag,
+		Label:     *labelFlag,
+		ProjectID: *projectIDFlag,
+		Location:  *locationFlag,
+	})
+}


Test patch:
diff --git a/tools/certgen/internal/internal_test.go b/tools/certgen/internal/internal_test.go
index d0d2e17f..b4dcc0aa 100644
--- a/tools/certgen/internal/internal_test.go
+++ b/tools/certgen/internal/internal_test.go
@@ -107,5 +107,4 @@ func TestBadValues(t *testing.T) {
 			}
 		})
 	}
-
 }
diff --git a/tools/reaper/internal/internal_test.go b/tools/reaper/internal/internal_test.go
new file mode 100644
index 00000000..90335058
--- /dev/null
+++ b/tools/reaper/internal/internal_test.go
@@ -0,0 +1,84 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"fmt"
+	containerpb "google.golang.org/genproto/googleapis/container/v1"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsOrphaned(t *testing.T) {
+	testCases := []struct {
+		age      time.Duration
+		label    string
+		cluster  *containerpb.Cluster
+		expected bool
+	}{
+		{
+			time.Second,
+			"ci",
+			&containerpb.Cluster{
+				Name:           "Orphaned",
+				CreateTime:     time.Now().Add(-1 * time.Hour).Format(time.RFC3339),
+				ResourceLabels: map[string]string{"ci": "ok"},
+			},
+			true,
+		},
+		{
+			time.Second,
+			"ci",
+			&containerpb.Cluster{
+				Name:           "Not Orphaned, Created in the Future",
+				CreateTime:     time.Now().Add(1 * time.Hour).Format(time.RFC3339),
+				ResourceLabels: map[string]string{"ci": "ok"},
+			},
+			false,
+		},
+		{
+			time.Second,
+			"ci",
+			&containerpb.Cluster{
+				Name:           "Not Orphaned, Created in Past but no Label",
+				CreateTime:     time.Now().Add(-1 * time.Hour).Format(time.RFC3339),
+				ResourceLabels: map[string]string{},
+			},
+			false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%s deleted= %t", tc.cluster.Name, tc.expected), func(t *testing.T) {
+			actual := isOrphaned(tc.cluster, &Params{
+				Age:   tc.age,
+				Label: tc.label,
+			})
+			if actual != tc.expected {
+				if tc.expected {
+					t.Errorf("Cluster %s was NOT marked as orphaned but should have.", tc.cluster.Name)
+				} else {
+					t.Errorf("Cluster %s was marked as orphaned but should not have.", tc.cluster.Name)
+				}
+			}
+		})
+	}
+}
+
+func TestSelfLinkToQualifiedName(t *testing.T) {
+	assert := assert.New(t)
+	assert.Equal("projects/jeremyedwards-gaming-dev/zones/us-central1-a/clusters/orphaned", selfLinkToQualifiedName("https://container.googleapis.com/v1/projects/jeremyedwards-gaming-dev/zones/us-central1-a/clusters/orphaned"))
+}