Repository: gruntwork-io/kubergrunt
Pull request #96 (2020-09-21 16:48:46)

Introduce `eks sync-core-components` subcommand
This implements the logic in https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html , which we currently use a python script baked in the EKS module for. By moving the logic to `kubergrunt`, we not only make the module more flexible (e.g., you don't need to update to latest version everytime a new k8s version is released), but the logic can also be used with other methods of launching EKS, including the open source tf module.

See https://github.com/gruntwork-io/terraform-aws-eks/issues/174 for more details.


Patch:
diff --git a/README.md b/README.md
index 6840810..adaae8f 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,7 @@ The following commands are available as part of `kubergrunt`:
     * [token](#token)
     * [oidc-thumbprint](#oidc-thumbprint)
     * [deploy](#deploy)
+    * [sync-core-components](#sync-core-components)
 1. [k8s](#k8s)
     * [wait-for-ingress](#wait-for-ingress)
     * [kubectl](#kubectl)
@@ -245,6 +246,25 @@ Currently `kubergrunt` does not implement any checks for these resources to be i
 plan to bake in checks into the deployment command to verify that all services have a disruption budget set, and warn
 the user of any services that do not have a check.
 
+#### sync-core-components
+
+This subcommand will sync the core components of an EKS cluster to match the deployed Kubernetes version by following
+the steps listed [in the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html).
+
+The core components managed by this command are:
+
+- kube-proxy
+- Amazon VPC CNI plug-in
+- CoreDNS
+
+By default, this command will rotate the images without waiting for the Pods to be redeployed. You can use the `--wait`
+option to force the command to wait until all the Pods have been replaced.
+
+Example:
+
+```bash
+kubergrunt eks sync-core-components --eks-cluster-arn EKS_CLUSTER_ARN
+```
 
 ### k8s
 
diff --git a/cmd/eks.go b/cmd/eks.go
index 9ee48ad..bef8b92 100644
--- a/cmd/eks.go
+++ b/cmd/eks.go
@@ -56,6 +56,11 @@ var (
 		Value: 15 * time.Second,
 		Usage: "The amount of time to sleep between retries as duration (e.g 10m = 10 minutes) for retry loops during the command. The total amount of time this command will try is based on max-retries and sleep-between-retries. Defaults to 15 seconds.",
 	}
+	waitTimeoutFlag = cli.StringFlag{
+		Name:  "wait-timeout",
+		Value: "10m",
+		Usage: "The amount of time to wait for operations to complete, expressed as a duration (e.g., 10m = 10 minutes). Defaults to 10 minutes.",
+	}
 
 	// Token related flags
 	clusterIDFlag = cli.StringFlag{
@@ -123,6 +128,26 @@ func SetupEksCommand() cli.Command {
 					oidcIssuerUrlFlag,
 				},
 			},
+			cli.Command{
+				Name:  "sync-core-components",
+				Usage: "Update the core Kubernetes applications deployed on to the EKS cluster to match the Kubernetes version.",
+				Description: `Update the core Kubernetes applications deployed on to an EKS cluster to ensure that the versions match with the expected versions deployed for the configured Kubernetes version.
+
+There are three core applications on an EKS cluster:
+    - kube-proxy
+    - coredns
+    - VPC CNI Plugin
+
+Each of these are managed in Kubernetes as DaemonSet, Deployment, and DaemonSet respectively. This command will use kubectl under the hood to patch the manifests to deploy the expected version based on what the current Kubernetes version is of the cluster. As such, this command should be run every time the Kubernetes version is updated on the EKS cluster.
+
+The versions deployed are based on what is listed in the official guide provided by AWS: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html`,
+				Action: syncClusterComponents,
+				Flags: []cli.Flag{
+					eksClusterArnFlag,
+					waitFlag,
+					waitTimeoutFlag,
+				},
+			},
 			cli.Command{
 				Name:  "deploy",
 				Usage: "Zero downtime roll out of cluster updates to worker nodes.",
@@ -284,3 +309,14 @@ func rollOutDeployment(cliContext *cli.Context) error {
 		waitSleepBetweenRetries,
 	)
 }
+
+// Command action for `kubergrunt eks sync-core-components`
+func syncClusterComponents(cliContext *cli.Context) error {
+	eksClusterArn, err := entrypoint.StringFlagRequiredE(cliContext, eksClusterArnFlag.Name)
+	if err != nil {
+		return err
+	}
+	shouldWait := cliContext.Bool(waitFlag.Name)
+	waitTimeout := cliContext.String(waitTimeoutFlag.Name)
+	return eks.SyncClusterComponents(eksClusterArn, shouldWait, waitTimeout)
+}
diff --git a/eks/errors.go b/eks/errors.go
index 8870b24..6849693 100644
--- a/eks/errors.go
+++ b/eks/errors.go
@@ -124,3 +124,23 @@ type NoPeerCertificatesError struct {
 func (err NoPeerCertificatesError) Error() string {
 	return fmt.Sprintf("Could not find any peer certificates for URL %s", err.URL)
 }
+
+// UnsupportedEKSVersion is returned when the Kubernetes version of the EKS cluster is not supported.
+type UnsupportedEKSVersion struct {
+	version string
+}
+
+func (err UnsupportedEKSVersion) Error() string {
+	return fmt.Sprintf("%s is not a supported version for kubergrunt eks upgrade. Please contact support@gruntwork.io for more info.", err.version)
+}
+
+// CoreComponentUnexpectedConfigurationErr error is returned when the EKS core components are in an unexpected
+// configuration, such as a different number of containers.
+type CoreComponentUnexpectedConfigurationErr struct {
+	component string
+	reason    string
+}
+
+func (err CoreComponentUnexpectedConfigurationErr) Error() string {
+	return fmt.Sprintf("Core component %s is in unexpected configuration: %s", err.component, err.reason)
+}
diff --git a/eks/fixture/aws-k8s-cni.yaml b/eks/fixture/aws-k8s-cni.yaml
new file mode 100644
index 0000000..8481ff0
--- /dev/null
+++ b/eks/fixture/aws-k8s-cni.yaml
@@ -0,0 +1,150 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aws-node
+rules:
+  - apiGroups:
+      - crd.k8s.amazonaws.com
+    resources:
+      - "*"
+    verbs:
+      - "*"
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+      - namespaces
+    verbs: ["list", "watch", "get"]
+  - apiGroups: ["extensions"]
+    resources:
+      - daemonsets
+    verbs: ["list", "watch"]
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-node
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aws-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node
+subjects:
+  - kind: ServiceAccount
+    name: aws-node
+    namespace: kube-system
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: aws-node
+  namespace: kube-system
+  labels:
+    k8s-app: aws-node
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: "10%"
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-node
+    spec:
+      priorityClassName: system-node-critical
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "beta.kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "beta.kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: eks.amazonaws.com/compute-type
+                    operator: NotIn
+                    values:
+                      - fargate
+      serviceAccountName: aws-node
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+      containers:
+        - image: 602401143452.dkr.ecr.ap-northeast-1.amazonaws.com/amazon-k8s-cni:v1.5.7
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 61678
+              name: metrics
+          name: aws-node
+          env:
+            - name: AWS_VPC_K8S_CNI_LOGLEVEL
+              value: DEBUG
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            requests:
+              cpu: 10m
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /host/var/log
+              name: log-dir
+            - mountPath: /var/run/docker.sock
+              name: dockersock
+            - mountPath: /var/run/dockershim.sock
+              name: dockershim
+      volumes:
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        - name: log-dir
+          hostPath:
+            path: /var/log
+        - name: dockersock
+          hostPath:
+            path: /var/run/docker.sock
+        - name: dockershim
+          hostPath:
+            path: /var/run/dockershim.sock
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: eniconfigs.crd.k8s.amazonaws.com
+spec:
+  scope: Cluster
+  group: crd.k8s.amazonaws.com
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  names:
+    plural: eniconfigs
+    singular: eniconfig
+    kind: ENIConfig
diff --git a/eks/sync.go b/eks/sync.go
new file mode 100644
index 0000000..730fd48
--- /dev/null
+++ b/eks/sync.go
@@ -0,0 +1,352 @@
+package eks
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/gruntwork-io/gruntwork-cli/collections"
+	"github.com/gruntwork-io/gruntwork-cli/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/gruntwork-io/kubergrunt/eksawshelper"
+	"github.com/gruntwork-io/kubergrunt/jsonpatch"
+	"github.com/gruntwork-io/kubergrunt/kubectl"
+	"github.com/gruntwork-io/kubergrunt/logging"
+)
+
+var (
+	// NOTE: Ensure that there is an entry for each supported version in the following tables.
+	supportedVersions = []string{"1.17", "1.16", "1.15", "1.14"}
+
+	coreDNSVersionLookupTable = map[string]string{
+		"1.17": "1.6.6-eksbuild.1",
+		"1.16": "1.6.6-eksbuild.1",
+		"1.15": "1.6.6-eksbuild.1",
+		"1.14": "1.6.6-eksbuild.1",
+	}
+
+	kubeProxyVersionLookupTable = map[string]string{
+		"1.17": "1.17.9-eksbuild.1",
+		"1.16": "1.16.13-eksbuild.1",
+		"1.15": "1.15.11-eksbuild.1",
+		"1.14": "1.14.9-eksbuild.1",
+	}
+
+	amazonVPCCNIVersionLookupTable = map[string]string{
+		"1.17": "1.6",
+		"1.16": "1.6",
+		"1.15": "1.6",
+		"1.14": "1.6",
+	}
+)
+
+const (
+	componentNamespace     = "kube-system"
+	kubeProxyDaemonSetName = "kube-proxy"
+	corednsDeploymentName  = "coredns"
+)
+
+// SyncClusterComponents will perform the steps described in
+// https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html
+// There are three core applications on an EKS cluster:
+//
+//    - kube-proxy
+//    - coredns
+//    - VPC CNI Plugin
+//
+// Each of these are managed in Kubernetes as DaemonSet, Deployment, and DaemonSet respectively. This command will use
+// the k8s API and kubectl command under the hood to patch the manifests to deploy the expected version based on what
+// the current Kubernetes version is of the cluster. As such, this command should be run every time the Kubernetes
+// version is updated on the EKS cluster.
+func SyncClusterComponents(
+	eksClusterArn string,
+	shouldWait bool,
+	waitTimeout string,
+) error {
+	logger := logging.GetProjectLogger()
+
+	logger.Info("Looking up deployed Kubernetes version")
+	clusterInfo, err := eksawshelper.GetClusterByArn(eksClusterArn)
+	if err != nil {
+		return err
+	}
+	k8sVersion := aws.StringValue(clusterInfo.Version)
+
+	if !collections.ListContainsElement(supportedVersions, k8sVersion) {
+		return errors.WithStackTrace(UnsupportedEKSVersion{k8sVersion})
+	}
+
+	kubeProxyVersion := kubeProxyVersionLookupTable[k8sVersion]
+	coreDNSVersion := coreDNSVersionLookupTable[k8sVersion]
+	amznVPCCNIVersion := amazonVPCCNIVersionLookupTable[k8sVersion]
+	logger.Info("Syncing Kubernetes Applications to:")
+	logger.Infof("\tkube-proxy:\t%s", kubeProxyVersion)
+	logger.Infof("\tcoredns:\t%s", coreDNSVersion)
+	logger.Infof("\tVPC CNI Plugin:\t%s", amznVPCCNIVersion)
+
+	kubectlOptions := &kubectl.KubectlOptions{EKSClusterArn: eksClusterArn}
+	clientset, err := kubectl.GetKubernetesClientFromOptions(kubectlOptions)
+	if err != nil {
+		return err
+	}
+
+	awsRegion, err := eksawshelper.GetRegionFromArn(eksClusterArn)
+	if err != nil {
+		return err
+	}
+	if err := upgradeKubeProxy(kubectlOptions, clientset, awsRegion, kubeProxyVersion, shouldWait, waitTimeout); err != nil {
+		return err
+	}
+	if err := upgradeCoreDNS(kubectlOptions, clientset, awsRegion, coreDNSVersion, shouldWait, waitTimeout); err != nil {
+		return err
+	}
+	if err := updateVPCCNI(kubectlOptions, awsRegion, amznVPCCNIVersion); err != nil {
+		return err
+	}
+
+	logger.Info("Successfully updated core components.")
+	return nil
+}
+
+// upgradeKubeProxy will update to the latest kube-proxy version if necessary. If shouldWait is set to true, this
+// routine will wait until the new images are fully rolled out before continuing.
+func upgradeKubeProxy(
+	kubectlOptions *kubectl.KubectlOptions,
+	clientset *kubernetes.Clientset,
+	awsRegion string,
+	kubeProxyVersion string,
+	shouldWait bool,
+	waitTimeout string,
+) error {
+	logger := logging.GetProjectLogger()
+
+	targetImage := fmt.Sprintf("602401143452.dkr.ecr.%s.amazonaws.com/eks/kube-proxy:v%s", awsRegion, kubeProxyVersion)
+	currentImage, err := getCurrentDeployedKubeProxyImage(clientset)
+	if err != nil {
+		return err
+	}
+	if currentImage == targetImage {
+		logger.Info("Current deployed version matches expected version. Skipping kube-proxy update.")
+		return nil
+	}
+
+	logger.Infof("Upgrading current deployed version of kube-proxy (%s) to match expected version (%s).", currentImage, targetImage)
+	if err := updateKubeProxyDaemonsetImage(clientset, targetImage); err != nil {
+		return err
+	}
+	if shouldWait {
+		logger.Info("Waiting until new image for kube-proxy is rolled out.")
+		// Ideally we will implement the following routine using the raw client-go library, but implementing this
+		// functionality directly on the API is fairly complex, and thus we rely on the built in mechanism in kubectl
+		// instead.
+		args := []string{
+			"rollout",
+			"status",
+			fmt.Sprintf("daemonset/%s", kubeProxyDaemonSetName),
+			"-n", componentNamespace,
+			"--timeout", waitTimeout,
+		}
+		return kubectl.RunKubectl(kubectlOptions, args...)
+	}
+	return nil
+}
+
+// getCurrentDeployedKubeProxyImage will return the currently configured kube-proxy image on the daemonset.
+func getCurrentDeployedKubeProxyImage(clientset *kubernetes.Clientset) (string, error) {
+	daemonset, err := clientset.AppsV1().DaemonSets(componentNamespace).Get(context.Background(), kubeProxyDaemonSetName, metav1.GetOptions{})
+	if err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	daemonsetContainers := daemonset.Spec.Template.Spec.Containers
+	if len(daemonsetContainers) != 1 {
+		err := CoreComponentUnexpectedConfigurationErr{
+			component: "kube-proxy",
+			reason:    fmt.Sprintf("unexpected number of containers (%d)", len(daemonsetContainers)),
+		}
+		return "", errors.WithStackTrace(err)
+	}
+	return daemonsetContainers[0].Image, nil
+}
+
+// updateKubeProxyDaemonsetImage will update the deployed kube-proxy DaemonSet to the specified target container image.
+func updateKubeProxyDaemonsetImage(clientset *kubernetes.Clientset, targetImage string) error {
+	patch := []jsonpatch.PatchString{
+		{
+			Op: jsonpatch.ReplaceOp,
+			// Patch the first container's image field
+			Path:  "/spec/template/spec/containers/0/image",
+			Value: targetImage,
+		},
+	}
+	patchOpJson, err := json.Marshal(patch)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	daemonsetAPI := clientset.AppsV1().DaemonSets(componentNamespace)
+	if _, err := daemonsetAPI.Patch(context.Background(), kubeProxyDaemonSetName, k8stypes.JSONPatchType, patchOpJson, metav1.PatchOptions{}); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	return nil
+}
+
+// upgradeCoreDNS will update to the latest coredns version if necessary. If shouldWait is set to true, this routine
+// will wait until the new images are fully rolled out before continuing.
+func upgradeCoreDNS(
+	kubectlOptions *kubectl.KubectlOptions,
+	clientset *kubernetes.Clientset,
+	awsRegion string,
+	coreDNSVersion string,
+	shouldWait bool,
+	waitTimeout string,
+) error {
+	logger := logging.GetProjectLogger()
+
+	targetImage := fmt.Sprintf("602401143452.dkr.ecr.%s.amazonaws.com/eks/coredns:v%s", awsRegion, coreDNSVersion)
+	currentImage, err := getCurrentDeployedCoreDNSImage(clientset)
+	if err != nil {
+		return err
+	}
+	if currentImage == targetImage {
+		logger.Info("Current deployed version matches expected version. Skipping coredns update.")
+		return nil
+	}
+
+	logger.Infof("Upgrading current deployed version of coredns (%s) to match expected version (%s).", currentImage, targetImage)
+	if err := updateCoreDNSDeploymentImage(clientset, targetImage); err != nil {
+		return err
+	}
+	if shouldWait {
+		logger.Info("Waiting until new image for coredns is rolled out.")
+		// Ideally we will implement the following routine using the raw client-go library, but implementing this
+		// functionality directly on the API is fairly complex, and thus we rely on the built in mechanism in kubectl
+		// instead.
+		args := []string{
+			"rollout",
+			"status",
+			fmt.Sprintf("deployment/%s", corednsDeploymentName),
+			"-n", componentNamespace,
+			"--timeout", waitTimeout,
+		}
+		return kubectl.RunKubectl(kubectlOptions, args...)
+	}
+	return nil
+}
+
+// getCurrentDeployedCoreDNSImage will return the currently configured coredns image on the deployment.
+func getCurrentDeployedCoreDNSImage(clientset *kubernetes.Clientset) (string, error) {
+	deployment, err := clientset.AppsV1().Deployments(componentNamespace).Get(context.Background(), corednsDeploymentName, metav1.GetOptions{})
+	if err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	deploymentContainers := deployment.Spec.Template.Spec.Containers
+	if len(deploymentContainers) != 1 {
+		err := CoreComponentUnexpectedConfigurationErr{
+			component: "coredns",
+			reason:    fmt.Sprintf("unexpected number of containers (%d)", len(deploymentContainers)),
+		}
+		return "", errors.WithStackTrace(err)
+	}
+	return deploymentContainers[0].Image, nil
+}
+
+// updateCoreDNSDeploymentImage will update the deployed coredns Deployment to the specified target container image.
+func updateCoreDNSDeploymentImage(clientset *kubernetes.Clientset, targetImage string) error {
+	patch := []jsonpatch.PatchString{
+		{
+			Op: jsonpatch.ReplaceOp,
+			// Patch the first container's image field
+			Path:  "/spec/template/spec/containers/0/image",
+			Value: targetImage,
+		},
+	}
+	patchOpJson, err := json.Marshal(patch)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	deploymentAPI := clientset.AppsV1().Deployments(componentNamespace)
+	if _, err := deploymentAPI.Patch(context.Background(), corednsDeploymentName, k8stypes.JSONPatchType, patchOpJson, metav1.PatchOptions{}); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	return nil
+}
+
+// updateVPCCNI will apply the manifest to deploy the latest patch release of the target AWS VPC CNI version. Ideally we
+// would implement this using the raw Kubernetes API, but the CNI manifest contains additional resources on top of the
+// daemonset, and thus it is better to apply the manifests directly using kubectl than to translate it into underlying
+// API calls.
+func updateVPCCNI(kubectlOptions *kubectl.KubectlOptions, region string, vpcCNIVersion string) error {
+	var manifestPath string
+
+	// Figure out the manifest URL based on region
+	// Reference: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html
+	baseURL := fmt.Sprintf("https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-%s/config/v%s/", vpcCNIVersion, vpcCNIVersion)
+	if strings.HasPrefix(region, "cn-") {
+		manifestPath = baseURL + "aws-k8s-cni-cn.yaml"
+	} else if region == "us-gov-east-1" {
+		manifestPath = baseURL + "aws-k8s-cni-us-gov-east-1.yaml"
+	} else if region == "us-gov-west-1" {
+		manifestPath = baseURL + "aws-k8s-cni-us-gov-west-1.yaml"
+	} else if region == "us-west-2" {
+		manifestPath = baseURL + "aws-k8s-cni.yaml"
+	} else {
+		// This is technically the same manifest as us-west-2, but we need to replace references to us-west-2 with the
+		// appropriate region, so we need to first download the manifest to a temporary dir and update the region before
+		// applying.
+		workingDir, err := ioutil.TempDir("", "kubergrunt-sync")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(workingDir)
+		manifestPath = filepath.Join(workingDir, "aws-k8s-cni.yaml")
+
+		manifestURL := baseURL + "aws-k8s-cni.yaml"
+		if err := downloadVPCCNIManifestAndUpdateRegion(manifestURL, manifestPath, region); err != nil {
+			return err
+		}
+	}
+	return kubectl.RunKubectl(kubectlOptions, "apply", "-f", manifestPath)
+}
+
+// downloadVPCCNIManifestAndUpdateRegion will download the VPC CNI Kubernetes manifest at the given URL, update the
+// region, and save it the provided path. The region is always us-west-2 in the manifest (see
+// https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html)
+func downloadVPCCNIManifestAndUpdateRegion(url string, fpath string, region string) error {
+	out, err := os.Create(fpath)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	defer resp.Body.Close()
+
+	// As we stream the body contents, update any references to the region. We use bufio.Scanner so that we read and
+	// write line by line, as reading by bytes risks reading a part of the region.
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if _, err := fmt.Fprintln(out, strings.ReplaceAll(line, "us-west-2", region)); err != nil {
+			return errors.WithStackTrace(err)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	return nil
+}
diff --git a/jsonpatch/jsonpatch.go b/jsonpatch/jsonpatch.go
new file mode 100644
index 0000000..e3e922d
--- /dev/null
+++ b/jsonpatch/jsonpatch.go
@@ -0,0 +1,19 @@
+// Package that defines useful structs for constructing JSON Patch operations.
+package jsonpatch
+
+type Operation string
+
+const (
+	AddOp     Operation = "add"
+	RemoveOp  Operation = "remove"
+	ReplaceOp Operation = "replace"
+	MoveOp    Operation = "move"
+	CopyOp    Operation = "copy"
+	TestOp    Operation = "test"
+)
+
+type PatchString struct {
+	Op    Operation `json:"op"`
+	Path  string    `json:"path"`
+	Value string    `json:"value"`
+}


Test patch:
diff --git a/eks/sync_test.go b/eks/sync_test.go
new file mode 100644
index 0000000..db5a17f
--- /dev/null
+++ b/eks/sync_test.go
@@ -0,0 +1,36 @@
+package eks
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDownloadVPCCNIManifestAndUpdateRegion(t *testing.T) {
+	t.Parallel()
+
+	workingDir, err := ioutil.TempDir("", "kubergrunt-sync")
+	require.NoError(t, err)
+	defer os.RemoveAll(workingDir)
+	manifestPath := filepath.Join(workingDir, "aws-k8s-cni.yaml")
+
+	require.NoError(
+		t,
+		downloadVPCCNIManifestAndUpdateRegion(
+			"https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/f5bac1d9ff4b7261d44d50705f3657b65f9dbdc5/config/v1.5/aws-k8s-cni.yaml",
+			manifestPath,
+			"ap-northeast-1",
+		),
+	)
+
+	// Compare the downloaded file against the fixture
+	expectedF, err := ioutil.ReadFile(filepath.Join(".", "fixture", "aws-k8s-cni.yaml"))
+	require.NoError(t, err)
+	actualF, err := ioutil.ReadFile(manifestPath)
+	require.NoError(t, err)
+	assert.Equal(t, expectedF, actualF)
+}