Repository: google/kf (go)

Issue:
Custom routing config to support future GKE versions
<!-- If you need to report a security issue with kf, send an email to kf-security@google.com. -->

## What problem is the feature trying to solve?
Future cluster versions of GKE (1.15.3-gke.19 and greater, 1.14.3-gke.12 and greater, 1.13.10-gke.8 and greater) have slightly different naming for the Istio gateway and namespace (see https://cloud.google.com/run/docs/gke/troubleshooting for more details)


## What is the feature?
Support routing features in Kf with these later versions of GKE via configmaps


## What impact would the feature have?

Enable routing config for Istio ingress

Pull request description:
Add support for routing configmap
<!-- Include the issue number below -->
Fixes #805 

## Proposed Changes

* Certain fields in virtualservices (used in routing) can now be configured using a configmap `config-routing.yaml`. This supports upcoming changes for fields such as the ingress name and namespace (see #805 for more details)

## Release Notes

<!-- kf follows the Keep A Changelog standard for release notes.

https://keepachangelog.com/en/1.0.0/

Changelog entries should be one per line and start with one of the following
words:

`Added` for new features.
`Changed` for changes in existing functionality.
`Deprecated` for soon-to-be removed features.
`Removed` for now removed features.
`Fixed` for any bug fixes.
`Security` in case of vulnerabilities.

If one of the changes is breaking include that as a second word e.g.

  Removed BREAKING support for manifests v1
-->

```release-note
Added support for routing configmap
```


Patch:
diff --git a/config/config-routing.yaml b/config/config-routing.yaml
new file mode 100644
index 0000000..e0acd82
--- /dev/null
+++ b/config/config-routing.yaml
@@ -0,0 +1,38 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-routing
+  namespace: kf
+data:
+  _example: |
+    ################################
+    #                              #
+    #    EXAMPLE CONFIGURATION     #
+    #                              #
+    ################################
+
+    # This block is not actually functional configuration,
+    # but serves to illustrate the available configuration
+    # options and document them in a way that is accessible
+    # to users that `kubectl edit` this config map.
+    #
+    # These sample configuration options may be copied out of
+    # this example block and unindented to be in the data block
+    # to actually change the configuration.
+    ingress.servicename: "istio-ingress"
+    ingress.namespace: "gke-system"
+    knative.ingress.gateway: "gke-system-gateway"
diff --git a/pkg/reconciler/base.go b/pkg/reconciler/base.go
index a84339b..b6d8581 100644
--- a/pkg/reconciler/base.go
+++ b/pkg/reconciler/base.go
@@ -41,6 +41,11 @@ import (
 	"knative.dev/pkg/logging/logkey"
 )
 
+// ConfigStore is a minimal interface to the config stores used by our controllers.
+type ConfigStore interface {
+	ToContext(ctx context.Context) context.Context
+}
+
 // Base implements the core controller logic, given a Reconciler.
 type Base struct {
 	// KubeClientSet allows us to talk to the k8s for core APIs
@@ -69,11 +74,6 @@ type Base struct {
 	SecretInformer informerscorev1.SecretInformer
 }
 
-// ConfigStore is a minimal interface to the config stores used by our controllers.
-type ConfigStore interface {
-	ToContext(ctx context.Context) context.Context
-}
-
 // NewBase instantiates a new instance of Base implementing
 // the common & boilerplate code between our reconcilers.
 func NewBase(ctx context.Context, cmw configmap.Watcher) *Base {
diff --git a/pkg/reconciler/integration/README.md b/pkg/reconciler/integration/README.md
new file mode 100644
index 0000000..609281e
--- /dev/null
+++ b/pkg/reconciler/integration/README.md
@@ -0,0 +1,53 @@
+## Kf Black Box Tests
+
+The goal of these black box tests is to be able to test specific K8s types
+without any knowledge about what they do; basically compliance tests.
+
+This package implements the following classes of tests (listed in order of importance):
+
+1. Defaulting validation: we ensure the defaults for a resource don't change by
+   creating it and comparing the values. This will catch unexpected changes in
+   defaulting webhooks. (This bitten us in 0.1 and 0.2)
+2. Health validation: we ensure the status of the object behaves consistently
+   with our expectations. (Ongoing issue)
+3. Update validation: ensure that the status of updated objects behaves in
+   accordance with our expectations.
+4. Upgrade validation: ensure that upgrading an object from one version to the
+   next goes as anticipated. Ideally this will be well-tested by the creators.
+
+NOTE: These black-box-tests are only intended to run positive examples, if we
+start expecting certain error messages, codes, etc. we're likely going to end up
+on the un-maintainable spectrum.
+
+## Algorithms
+
+The following are suggested algorithms to implement the validation tests.
+
+### Defaulting Validation Algorithm
+
+1. Create an object
+2. Ensure no additional fields have been populated on the response in the
+   `spec` or parts of `metadata`.
+
+### Health Validation Algorithm
+
+1. Create an object
+2. Watch each update until a successful condition has been met. Success should
+   be defined by Rego expression.
+3. Run Rego expression(s) for each update we receive. If an expression fails,
+   the whole test should fail. These expressions SHOULD test that our
+   interpretation of the status as a state machine is correct.
+4. If some timeout occurs before the resource is ready, fail the test.
+
+### Update Validation Algorithm
+
+This is the same as the health validation algorithm, but after the initial
+success, the `spec` should be modified and the test will run the health
+validation again.
+
+### Upgrade Validation Algorithm
+
+1. With an object, ask Kuberntes to upgrade it from version X to Y to produce Z
+2. Ask Kubernetes to downgrade Z from version Y to X.
+3. Assert that the specs are unchanged and that both upgrade and downgrade
+   were successful.
diff --git a/pkg/reconciler/integration/doc.go b/pkg/reconciler/integration/doc.go
new file mode 100644
index 0000000..363bb8e
--- /dev/null
+++ b/pkg/reconciler/integration/doc.go
@@ -0,0 +1,17 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package integration contains integration tests for the reconciler and
+// its dependencies.
+package integration
diff --git a/pkg/reconciler/route/config/routing.go b/pkg/reconciler/route/config/routing.go
new file mode 100644
index 0000000..5585c95
--- /dev/null
+++ b/pkg/reconciler/route/config/routing.go
@@ -0,0 +1,79 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	RoutingConfigName       = "config-routing"
+	KnativeServingNamespace = "knative-serving"
+
+	IngressServiceNameKey    = "ingress.servicename"
+	IngressNamespaceKey      = "ingress.namespace"
+	KnativeIngressGatewayKey = "knative.ingress.gateway"
+
+	DefaultIngressServiceName    = "cluster-local-gateway"
+	DefaultIngressNamespace      = "gke-system"
+	DefaultKnativeIngressGateway = "gke-system-gateway"
+)
+
+// RoutingConfig contains the networking configuration defined in the
+// network config map.
+type RoutingConfig struct {
+	// IngressServiceName is the name of the Istio Ingress svc
+	IngressServiceName string
+
+	// IngressNamespace is the K8s namespace to search for Ingresses
+	IngressNamespace string
+
+	// KnativeIngressGateway is the name of ingress gateway in knative-serving namespace
+	KnativeIngressGateway string
+}
+
+// NewRoutingConfigFromConfigMap creates a RoutingConfig from the supplied ConfigMap
+func NewRoutingConfigFromConfigMap(configMap *corev1.ConfigMap) (*RoutingConfig, error) {
+	rc := &RoutingConfig{}
+
+	if ingressServiceName, ok := configMap.Data[IngressServiceNameKey]; !ok {
+		rc.IngressServiceName = DefaultIngressServiceName
+	} else {
+		rc.IngressServiceName = ingressServiceName
+	}
+
+	if ingressNamespace, ok := configMap.Data[IngressNamespaceKey]; !ok {
+		rc.IngressNamespace = DefaultIngressNamespace
+	} else {
+		rc.IngressNamespace = ingressNamespace
+	}
+
+	if knativeIngressGateway, ok := configMap.Data[KnativeIngressGatewayKey]; !ok {
+		rc.KnativeIngressGateway = DefaultKnativeIngressGateway
+	} else {
+		rc.KnativeIngressGateway = knativeIngressGateway
+	}
+
+	rc.KnativeIngressGateway = fmt.Sprintf("%s/%s", KnativeServingNamespace, rc.KnativeIngressGateway)
+	return rc, nil
+}
+
+// GatewayHost is the name of gateway set as route destination host in the kf virtualservice
+// (usually back to the ingress gateway or cluster local gateway)
+func (rc *RoutingConfig) GatewayHost() string {
+	return fmt.Sprintf("%s.%s.svc.cluster.local", rc.IngressServiceName, rc.IngressNamespace)
+}
diff --git a/pkg/reconciler/route/config/store.go b/pkg/reconciler/route/config/store.go
new file mode 100644
index 0000000..47c3f31
--- /dev/null
+++ b/pkg/reconciler/route/config/store.go
@@ -0,0 +1,90 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"knative.dev/pkg/configmap"
+)
+
+type cfgKey struct{}
+
+type Config struct {
+	Routing *RoutingConfig
+}
+
+func FromContext(ctx context.Context) *Config {
+	return ctx.Value(cfgKey{}).(*Config)
+}
+
+func ToContext(ctx context.Context, c *Config) context.Context {
+	return context.WithValue(ctx, cfgKey{}, c)
+}
+
+// Store is based on configmap.UntypedStore and is used to store and watch for
+// updates to configuration related to routes.
+type Store struct {
+	*configmap.UntypedStore
+}
+
+// NewStore creates a configmap.UntypedStore based config store.
+//
+// logger must be non-nil implementation of configmap.Logger (commonly used
+// loggers conform)
+//
+// onAfterStore is a variadic list of callbacks to run
+// after the ConfigMap has been processed and stored.
+//
+// See also: configmap.NewUntypedStore().
+func NewStore(logger configmap.Logger, onAfterStore ...func(name string, value interface{})) *Store {
+	store := &Store{
+		UntypedStore: configmap.NewUntypedStore(
+			"routing",
+			logger,
+			configmap.Constructors{
+				RoutingConfigName: NewRoutingConfigFromConfigMap,
+			},
+			onAfterStore...,
+		),
+	}
+
+	return store
+}
+
+// NewDefaultConfigStore creates a routing config store populated
+// with default keys and values.
+func NewDefaultConfigStore(logger configmap.Logger) *Store {
+	store := NewStore(logger)
+	routingConfig := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: RoutingConfigName,
+		},
+	}
+	store.OnConfigChanged(routingConfig)
+	return store
+}
+
+func (s *Store) ToContext(ctx context.Context) context.Context {
+	return ToContext(ctx, s.Load())
+}
+
+func (s *Store) Load() *Config {
+	return &Config{
+		Routing: s.UntypedLoad(RoutingConfigName).(*RoutingConfig),
+	}
+}
diff --git a/pkg/reconciler/route/controller.go b/pkg/reconciler/route/controller.go
index 4627d9d..85c8788 100644
--- a/pkg/reconciler/route/controller.go
+++ b/pkg/reconciler/route/controller.go
@@ -25,6 +25,7 @@ import (
 	kflisters "github.com/google/kf/pkg/client/listers/kf/v1alpha1"
 	"github.com/google/kf/pkg/reconciler"
 	appresources "github.com/google/kf/pkg/reconciler/app/resources"
+	"github.com/google/kf/pkg/reconciler/route/config"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/cache"
@@ -70,6 +71,26 @@ func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl
 		Handler:    controller.HandleAll(logError(logger, EnqueueRoutesOfVirtualService(enqueue, c.routeLister))),
 	})
 
+	logger.Info("Setting up ConfigMap receivers")
+	configsToResync := []interface{}{
+		&config.RoutingConfig{},
+	}
+	resync := configmap.TypeFilter(configsToResync...)(func(string, interface{}) {
+		// When configmap updates, queue all existing routes and routeclaims in the form of namespaced RouteSpecFields
+		routes := routeInformer.Informer().GetStore().List()
+		routeClaims := routeClaimInformer.Informer().GetStore().List()
+		for _, rc := range routeClaims {
+			enqueue(rc)
+		}
+
+		for _, r := range routes {
+			enqueue(r)
+		}
+	})
+	configStore := config.NewStore(logger.Named("routing-config-store"), resync)
+	configStore.WatchConfigs(cmw)
+	c.configStore = configStore
+
 	return impl
 }
 
diff --git a/pkg/reconciler/route/default_config_logger.go b/pkg/reconciler/route/default_config_logger.go
new file mode 100644
index 0000000..7626f4a
--- /dev/null
+++ b/pkg/reconciler/route/default_config_logger.go
@@ -0,0 +1,47 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package route
+
+import (
+	"fmt"
+	"io"
+)
+
+// DefaultConfigLogger is a basic logger used in tests when a *testing.T type
+// is not available (such as in Examples)
+type DefaultConfigLogger struct {
+	w io.Writer
+}
+
+// Infof does not print out to writer since doing so would modify the output in the Examples
+func (l *DefaultConfigLogger) Infof(format string, v ...interface{}) {
+	return
+}
+
+func (l *DefaultConfigLogger) Fatalf(format string, v ...interface{}) {
+	fmt.Fprintf(l.w, format, v...)
+}
+
+func (l *DefaultConfigLogger) Errorf(format string, v ...interface{}) {
+	fmt.Fprintf(l.w, format, v...)
+}
+
+// NewDefaultConfigLogger creates an instance of a DefaultConfigLogger
+func NewDefaultConfigLogger(w io.Writer) *DefaultConfigLogger {
+	return &DefaultConfigLogger{
+		w: w,
+	}
+}
diff --git a/pkg/reconciler/route/reconciler.go b/pkg/reconciler/route/reconciler.go
index 32d1401..2fa9213 100644
--- a/pkg/reconciler/route/reconciler.go
+++ b/pkg/reconciler/route/reconciler.go
@@ -44,6 +44,7 @@ type Reconciler struct {
 	routeLister          kflisters.RouteLister
 	routeClaimLister     kflisters.RouteClaimLister
 	virtualServiceLister istiolisters.VirtualServiceLister
+	configStore          reconciler.ConfigStore
 }
 
 // Check that our Reconciler implements controller.Reconciler
@@ -59,6 +60,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
 
 	logger := logging.FromContext(ctx).With("namespace", route.Namespace)
 
+	ctx = r.configStore.ToContext(ctx)
+
 	if r.IsNamespaceTerminating(route.Namespace) {
 		logger.Errorf("skipping sync for route %#v", route)
 		return nil
@@ -132,7 +135,7 @@ func (r *Reconciler) ApplyChanges(
 		return err
 	}
 
-	desired, err := resources.MakeVirtualService(claims, routes)
+	desired, err := resources.MakeVirtualService(ctx, claims, routes)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/reconciler/route/resources/virtual_service.go b/pkg/reconciler/route/resources/virtual_service.go
index c5bfb3c..7c54041 100644
--- a/pkg/reconciler/route/resources/virtual_service.go
+++ b/pkg/reconciler/route/resources/virtual_service.go
@@ -15,6 +15,7 @@
 package resources
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -22,6 +23,7 @@ import (
 	"sort"
 
 	"github.com/google/kf/pkg/apis/kf/v1alpha1"
+	"github.com/google/kf/pkg/reconciler/route/config"
 	"github.com/google/kf/third_party/knative-serving/pkg/network"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -47,7 +49,7 @@ func MakeVirtualServiceLabels(spec v1alpha1.RouteSpecFields) map[string]string {
 }
 
 // MakeVirtualService creates a VirtualService from a Route object.
-func MakeVirtualService(claims []*v1alpha1.RouteClaim, routes []*v1alpha1.Route) (*networking.VirtualService, error) {
+func MakeVirtualService(ctx context.Context, claims []*v1alpha1.RouteClaim, routes []*v1alpha1.Route) (*networking.VirtualService, error) {
 	if len(claims) == 0 {
 		return nil, errors.New("claims must not be empty")
 	}
@@ -64,7 +66,7 @@ func MakeVirtualService(claims []*v1alpha1.RouteClaim, routes []*v1alpha1.Route)
 
 	// Build map of paths to set of bound apps
 	pathApps := buildPathApps(claims, routes)
-	httpRoutes, err := buildHTTPRoutes(hostDomain, pathApps, namespace)
+	httpRoutes, err := buildHTTPRoutes(ctx, hostDomain, pathApps, namespace)
 	if err != nil {
 		return nil, err
 	}
@@ -88,7 +90,7 @@ func MakeVirtualService(claims []*v1alpha1.RouteClaim, routes []*v1alpha1.Route)
 			},
 		},
 		Spec: networking.VirtualServiceSpec{
-			Gateways: []string{KnativeIngressGateway},
+			Gateways: []string{config.FromContext(ctx).Routing.KnativeIngressGateway},
 			Hosts:    []string{hostDomain},
 			HTTP:     httpRoutes,
 		},
@@ -97,7 +99,7 @@ func MakeVirtualService(claims []*v1alpha1.RouteClaim, routes []*v1alpha1.Route)
 
 // Create HTTP routes for all paths with the same host + domain.
 // Paths that do not have an app bound to them will return a 503 when a request is sent to that path.
-func buildHTTPRoutes(hostDomain string, pathApps map[string]sets.String, namespace string) ([]networking.HTTPRoute, error) {
+func buildHTTPRoutes(ctx context.Context, hostDomain string, pathApps map[string]sets.String, namespace string) ([]networking.HTTPRoute, error) {
 	var httpRoutes []networking.HTTPRoute
 
 	for path, apps := range pathApps {
@@ -118,13 +120,13 @@ func buildHTTPRoutes(hostDomain string, pathApps map[string]sets.String, namespa
 						HTTPStatus: http.StatusServiceUnavailable,
 					},
 				},
-				Route: buildDefaultRouteDestination(),
+				Route: buildDefaultRouteDestination(ctx),
 			}
 		} else {
 			// create HTTP route for path with app(s) bound
 			httpRoute = networking.HTTPRoute{
 				Match:   pathMatchers,
-				Route:   buildRouteDestinations(apps.List(), namespace),
+				Route:   buildRouteDestinations(ctx, apps.List(), namespace),
 				Headers: buildForwardingHeaders(hostDomain),
 			}
 		}
@@ -141,14 +143,14 @@ func buildHTTPRoutes(hostDomain string, pathApps map[string]sets.String, namespa
 // Hostname + domain + path combos with bound app(s) have a custom route destination for each path.
 // The request is sent back to the istio ingress gateway with the host set as the app's internal host name.
 // If there are multiple apps bound to a route, the traffic is split uniformly across the apps.
-func buildRouteDestinations(appNames []string, namespace string) []networking.HTTPRouteDestination {
+func buildRouteDestinations(ctx context.Context, appNames []string, namespace string) []networking.HTTPRouteDestination {
 	routeWeights := getRouteWeights(len(appNames))
 	routeDestinations := []networking.HTTPRouteDestination{}
 
 	for i, app := range appNames {
 		routeDestination := networking.HTTPRouteDestination{
 			Destination: networking.Destination{
-				Host: GatewayHost,
+				Host: config.FromContext(ctx).Routing.GatewayHost(),
 			},
 			Headers: buildHostHeader(app, namespace),
 			Weight:  routeWeights[i],
@@ -161,11 +163,11 @@ func buildRouteDestinations(appNames []string, namespace string) []networking.HT
 
 // Hostname + domain + path combos without an app are given the default route destination,
 // which simply redirects the request back to the istio ingress gateway
-func buildDefaultRouteDestination() []networking.HTTPRouteDestination {
+func buildDefaultRouteDestination(ctx context.Context) []networking.HTTPRouteDestination {
 	return []networking.HTTPRouteDestination{
 		{
 			Destination: networking.Destination{
-				Host: GatewayHost,
+				Host: config.FromContext(ctx).Routing.GatewayHost(),
 			},
 			Weight: 100,
 		},


Test patch:
diff --git a/pkg/kf/commands/apps/integration_test.go b/pkg/kf/commands/apps/integration_test.go
index 266092b..5daf1bb 100644
--- a/pkg/kf/commands/apps/integration_test.go
+++ b/pkg/kf/commands/apps/integration_test.go
@@ -40,6 +40,8 @@ const (
 	appTimeout = 90 * time.Second
 )
 
+var currentPort int = 8086
+
 // TestIntegration_Push pushes the echo app, lists it to ensure it can find a
 // domain, uses the proxy command and then posts to it. It finally deletes the
 // app.
@@ -55,7 +57,7 @@ func TestIntegration_Push(t *testing.T) {
 			"--path", filepath.Join(RootDir(ctx, t), "./samples/apps/echo"),
 		)
 		defer kf.Delete(ctx, appName)
-		checkEchoApp(ctx, t, kf, appName, 8080, ExpectedAddr(appName, ""))
+		checkEchoApp(ctx, t, kf, appName, getNextPort(), ExpectedAddr(appName, ""))
 	})
 }
 
@@ -74,7 +76,7 @@ func TestIntegration_Push_update(t *testing.T) {
 			"--path", filepath.Join(RootDir(ctx, t), "./samples/apps/echo"),
 		)
 		defer kf.Delete(ctx, appName)
-		checkEchoApp(ctx, t, kf, appName, 8087, ExpectedAddr(appName, ""))
+		checkEchoApp(ctx, t, kf, appName, getNextPort(), ExpectedAddr(appName, ""))
 
 		kf.Push(ctx, appName,
 			"--path", filepath.Join(RootDir(ctx, t), "./samples/apps/helloworld"),
@@ -82,7 +84,7 @@ func TestIntegration_Push_update(t *testing.T) {
 		// BUG(730): it takes a moment after the app becomes ready to reconcile the
 		// routes, the app is accessible but still points to the old one.
 		time.Sleep(45 * time.Second)
-		checkHelloWorldApp(ctx, t, kf, appName, 8088, ExpectedAddr(appName, ""))
+		checkHelloWorldApp(ctx, t, kf, appName, getNextPort(), ExpectedAddr(appName, ""))
 	})
 }
 
@@ -101,7 +103,7 @@ func TestIntegration_Push_docker(t *testing.T) {
 			"--docker-image=gcr.io/kf-releases/echo-app",
 		)
 		defer kf.Delete(ctx, appName)
-		checkEchoApp(ctx, t, kf, appName, 8086, ExpectedAddr(appName, ""))
+		checkEchoApp(ctx, t, kf, appName, getNextPort(), ExpectedAddr(appName, ""))
 	})
 }
 
@@ -118,7 +120,7 @@ func TestIntegration_Push_dockerfile(t *testing.T) {
 		kf.Push(ctx, appName, "--path", appPath, "--dockerfile", "Dockerfile")
 		defer kf.Delete(ctx, appName)
 
-		checkHelloWorldApp(ctx, t, kf, appName, 8089, ExpectedAddr(appName, ""))
+		checkHelloWorldApp(ctx, t, kf, appName, getNextPort(), ExpectedAddr(appName, ""))
 	})
 }
 
@@ -148,10 +150,13 @@ func TestIntegration_StopStart(t *testing.T) {
 
 		// TODO(#441): Use port 0 so that we don't have to worry about port
 		// collisions.
-		go kf.Proxy(ctx, appName, 8085)
+		port := getNextPort()
+		go kf.Proxy(ctx, appName, port)
+
+		url := fmt.Sprintf("http://localhost:%d", port)
 
 		{
-			resp, respCancel := RetryPost(ctx, t, "http://localhost:8085", appTimeout, http.StatusOK, "testing")
+			resp, respCancel := RetryPost(ctx, t, url, appTimeout, http.StatusOK, "testing")
 			defer resp.Body.Close()
 			defer respCancel()
 			Logf(t, "done hitting echo app to ensure it's working.")
@@ -163,7 +168,7 @@ func TestIntegration_StopStart(t *testing.T) {
 
 		{
 			Logf(t, "hitting echo app to ensure it's NOT working...")
-			resp, respCancel := RetryPost(ctx, t, "http://localhost:8085", appTimeout, http.StatusNotFound, "testing")
+			resp, respCancel := RetryPost(ctx, t, url, appTimeout, http.StatusNotFound, "testing")
 			defer resp.Body.Close()
 			defer respCancel()
 			Logf(t, "done hitting echo app to ensure it's NOT working.")
@@ -175,7 +180,7 @@ func TestIntegration_StopStart(t *testing.T) {
 
 		{
 			Logf(t, "hitting echo app to ensure it's working...")
-			resp, respCancel := RetryPost(ctx, t, "http://localhost:8085", 5*time.Minute, http.StatusOK, "testing")
+			resp, respCancel := RetryPost(ctx, t, url, 5*time.Minute, http.StatusOK, "testing")
 			defer resp.Body.Close()
 			defer respCancel()
 			Logf(t, "done hitting echo app to ensure it's working.")
@@ -206,7 +211,7 @@ func TestIntegration_Push_manifest(t *testing.T) {
 		)
 		defer kf.Delete(ctx, appName)
 
-		checkEnvApp(ctx, t, kf, appName, 8082, map[string]string{
+		checkEnvApp(ctx, t, kf, appName, getNextPort(), map[string]string{
 			"WHATNOW": "BROWNCOW",
 		}, ExpectedAddr(appName, ""))
 	})
@@ -302,7 +307,8 @@ func TestIntegration_Envs(t *testing.T) {
 		// This is only in place for cleanup if the test fails.
 		defer kf.Delete(ctx, appName)
 
-		checkEnvApp(ctx, t, kf, appName, 8081, map[string]string{
+		port := getNextPort()
+		checkEnvApp(ctx, t, kf, appName, port, map[string]string{
 			"ENV1": "VALUE1", // Set on push
 			"ENV2": "VALUE2", // Set on push
 		}, ExpectedAddr(appName, ""))
@@ -316,7 +322,7 @@ func TestIntegration_Envs(t *testing.T) {
 			// Overwrite ENV2 via set-env
 			RetryOnPanic(ctx, t, func() { kf.SetEnv(ctx, appName, "ENV2", "OVERWRITE2") })
 
-			assertVars(ctx, t, kf, appName, 8081, map[string]string{
+			assertVars(ctx, t, kf, appName, port, map[string]string{
 				"ENV2": "OVERWRITE2", // Set on push and overwritten via set-env
 			}, []string{"ENV1"})
 		})
@@ -349,13 +355,15 @@ func TestIntegration_Logs(t *testing.T) {
 
 		// TODO(#441): Use port 0 so that we don't have to worry about port
 		// collisions.
-		go kf.Proxy(ctx, appName, 8083)
+		port := getNextPort()
+		go kf.Proxy(ctx, appName, port)
 
 		// Write out an expected log line until the test dies. We do this more
 		// than once because we can't guarantee much about logs.
 		expectedLogLine := fmt.Sprintf("testing-%d", time.Now().UnixNano())
+		url := fmt.Sprintf("http://localhost:%d", port)
 		for i := 0; i < 10; i++ {
-			resp, respCancel := RetryPost(ctx, t, "http://localhost:8083", appTimeout, http.StatusOK, expectedLogLine)
+			resp, respCancel := RetryPost(ctx, t, url, appTimeout, http.StatusOK, expectedLogLine)
 			resp.Body.Close()
 			respCancel()
 		}
@@ -421,7 +429,7 @@ func TestIntegration_CfIgnore(t *testing.T) {
 		)
 		defer kf.Delete(ctx, appName)
 
-		checkCfignoreApp(ctx, t, kf, appName, 8084, ExpectedAddr(appName, ""))
+		checkCfignoreApp(ctx, t, kf, appName, getNextPort(), ExpectedAddr(appName, ""))
 	})
 }
 
@@ -665,3 +673,8 @@ func checkEnvApp(
 		assertVars(ctx, t, kf, appName, proxyPort, expectedVars, nil)
 	})
 }
+
+func getNextPort() int {
+	currentPort++
+	return currentPort
+}
diff --git a/pkg/kf/commands/routes/integration_test.go b/pkg/kf/commands/routes/integration_test.go
index 90863b4..c608a86 100644
--- a/pkg/kf/commands/routes/integration_test.go
+++ b/pkg/kf/commands/routes/integration_test.go
@@ -28,6 +28,8 @@ import (
 	. "github.com/google/kf/pkg/kf/testutil"
 )
 
+var currentPort int = 8082
+
 // TestIntegration_Routes creates a route via `create-route`, verifies it with
 // `routes`, deletes it via `delete-route` and then verifies again.
 func TestIntegration_Routes(t *testing.T) {
@@ -56,10 +58,12 @@ func TestIntegration_UnmappedRoute(t *testing.T) {
 		routeHost := fmt.Sprintf("%s.%s", hostname, domain)
 		findRoute(ctx, t, kf, hostname, domain, path, true)
 
-		go kf.ProxyRoute(ctx, routeHost, 8083)
+		port := getNextPort()
+		go kf.ProxyRoute(ctx, routeHost, port)
 
 		{
-			resp, respCancel := RetryGet(ctx, t, "http://localhost:8083/"+path, 90*time.Second, http.StatusServiceUnavailable)
+			url := fmt.Sprintf("http://localhost:%d/%s", port, path)
+			resp, respCancel := RetryGet(ctx, t, url, 90*time.Second, http.StatusServiceUnavailable)
 			defer resp.Body.Close()
 			defer respCancel()
 			Logf(t, "testing for 503")
@@ -91,10 +95,13 @@ func TestIntegration_MapRoute(t *testing.T) {
 		findRoute(ctx, t, kf, hostname, domain, path, true)
 
 		kf.MapRoute(ctx, appName, domain, "--hostname="+hostname, "--path="+path)
-		go kf.ProxyRoute(ctx, routeHost, 8083)
+
+		port := getNextPort()
+		go kf.ProxyRoute(ctx, routeHost, port)
 
 		{
-			resp, respCancel := RetryGet(ctx, t, "http://localhost:8083/"+path, 90*time.Second, http.StatusOK)
+			url := fmt.Sprintf("http://localhost:%d/%s", port, path)
+			resp, respCancel := RetryGet(ctx, t, url, 90*time.Second, http.StatusOK)
 			defer resp.Body.Close()
 			defer respCancel()
 			Logf(t, "testing for 200")
@@ -135,12 +142,14 @@ func TestIntegration_MultipleAppsPerRoute(t *testing.T) {
 		kf.MapRoute(ctx, helloWorldApp, domain, "--hostname="+hostname, "--path="+path)
 		kf.MapRoute(ctx, envsApp, domain, "--hostname="+hostname, "--path="+path)
 
-		go kf.ProxyRoute(ctx, routeHost, 8083)
+		port := getNextPort()
+		go kf.ProxyRoute(ctx, routeHost, port)
 
 		helloWorldCount := 0
 		envsCount := 0
+		url := fmt.Sprintf("http://localhost:%d/%s", port, path)
 		for i := 0; i < 100; i++ {
-			resp, respCancel := RetryGet(ctx, t, "http://localhost:8083/"+path, 90*time.Second, http.StatusOK)
+			resp, respCancel := RetryGet(ctx, t, url, 90*time.Second, http.StatusOK)
 			defer resp.Body.Close()
 			defer respCancel()
 			if resp.StatusCode == http.StatusOK {
@@ -192,3 +201,8 @@ func findRoute(ctx context.Context, t *testing.T, kf *Kf, hostname, domain, path
 		}
 	})
 }
+
+func getNextPort() int {
+	currentPort++
+	return currentPort
+}
diff --git a/pkg/kf/testutil/integration.go b/pkg/kf/testutil/integration.go
index c0e85bb..1d2fc5e 100644
--- a/pkg/kf/testutil/integration.go
+++ b/pkg/kf/testutil/integration.go
@@ -33,6 +33,16 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"knative.dev/pkg/injection"
 )
 
 const (
@@ -52,25 +62,36 @@ func DockerRegistry() string {
 	return os.Getenv(EnvDockerRegistry)
 }
 
-// RunIntegrationTest skips the tests if testing.Short() is true (via --short
-// flag) or if DOCKER_REGISTRY is not set. Otherwise it runs the given test.
-func RunIntegrationTest(t *testing.T, test func(ctx context.Context, t *testing.T)) {
+// ShouldSkipIntegration returns true if integration tests are being skipped.
+func ShouldSkipIntegration(t *testing.T) bool {
+	t.Helper()
+
+	if skipIntegration := os.Getenv("SKIP_INTEGRATION"); skipIntegration == "true" {
+		t.Skipf("Skipping %s because SKIP_INTEGRATION was true", t.Name())
+		return true
+	}
+
 	if !strings.HasPrefix(t.Name(), "TestIntegration_") {
 		// We want to enforce a convention so scripts can single out
 		// integration tests.
 		t.Fatalf("Integration tests must have the name format of 'TestIntegration_XXX`")
+		return true
 	}
 
-	t.Helper()
 	if testing.Short() {
-		t.Skip()
+		t.Skipf("Skipping %s because short tests were requested", t.Name())
+		return true
 	}
 
-	projID := os.Getenv(EnvDockerRegistry)
-	if projID == "" {
+	if registry := DockerRegistry(); registry == "" {
 		t.Skipf("%s is required for integration tests... Skipping...", EnvDockerRegistry)
+		return true
 	}
 
+	return false
+}
+
+func logTestResults(t *testing.T, f func()) {
 	start := time.Now()
 	defer func() {
 		state := "PASSED"
@@ -80,17 +101,132 @@ func RunIntegrationTest(t *testing.T, test func(ctx context.Context, t *testing.
 		t.Logf("Test %s took %v and %s", t.Name(), time.Since(start), state)
 	}()
 
+	f()
+}
+
+func withSignalCaptureCancel(t *testing.T, f func(ctx context.Context)) {
 	// Setup context that will allow us to cleanup if the user wants to
 	// cancel the tests.
 	ctx, cancel := context.WithCancel(context.Background())
-
 	// Give everything time to clean up.
 	defer time.Sleep(time.Second)
 	defer cancel()
 	CancelOnSignal(ctx, cancel, t.Log)
-	t.Log()
 
-	test(ctx, t)
+	f(ctx)
+}
+
+// RunIntegrationTest skips the tests if testing.Short() is true (via --short
+// flag) or if DOCKER_REGISTRY is not set. Otherwise it runs the given test.
+func RunIntegrationTest(t *testing.T, test func(ctx context.Context, t *testing.T)) {
+	t.Helper()
+
+	if ShouldSkipIntegration(t) {
+		return
+	}
+
+	logTestResults(t, func() {
+		withSignalCaptureCancel(t, func(ctx context.Context) {
+			test(ctx, t)
+		})
+	})
+}
+
+// RunKubeAPITest is for executing tests against the Kubernetes API directly.
+func RunKubeAPITest(t *testing.T, test func(ctx context.Context, t *testing.T)) {
+	t.Helper()
+
+	if ShouldSkipIntegration(t) {
+		return
+	}
+
+	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+	clientCfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+	restCfg, err := clientCfg.ClientConfig()
+	if err != nil {
+		t.Fatalf("couldn't fetch kubeconfig: %v", err)
+	}
+
+	logTestResults(t, func() {
+		withSignalCaptureCancel(t, func(ctx context.Context) {
+			ctx = contextWithRestConfig(ctx, restCfg)
+
+			// Set up the autowired informers so they can be used in testing
+			ctx, _ = injection.Default.SetupInformers(ctx, restCfg)
+
+			test(ctx, t)
+		})
+	})
+}
+
+type restConfigKey struct{}
+
+func contextWithRestConfig(ctx context.Context, cfg *rest.Config) context.Context {
+	return context.WithValue(ctx, restConfigKey{}, cfg)
+}
+
+func restConfigFromContext(ctx context.Context) *rest.Config {
+	return ctx.Value(restConfigKey{}).(*rest.Config)
+}
+
+// WithRestConfig gets the rest config from the context and passes it to the callback.
+func WithRestConfig(ctx context.Context, t *testing.T, callback func(cfg *rest.Config)) {
+	t.Helper()
+
+	callback(restConfigFromContext(ctx))
+}
+
+// WithKubernetes creates a Kubernetes client from the config on the context
+// and passes it to the callback.
+func WithKubernetes(ctx context.Context, t *testing.T, callback func(k8s kubernetes.Interface)) {
+	t.Helper()
+
+	WithRestConfig(ctx, t, func(cfg *rest.Config) {
+		k8s, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			t.Fatalf("Creating Kubernetes: %v", err)
+		}
+
+		callback(k8s)
+	})
+}
+
+// WithDynamicClient creates a dynamic Kubernetes client from the config on the
+// context and passes it to the callback.
+func WithDynamicClient(ctx context.Context, t *testing.T, callback func(client dynamic.Interface)) {
+	t.Helper()
+
+	WithRestConfig(ctx, t, func(cfg *rest.Config) {
+		dynamic, err := dynamic.NewForConfig(cfg)
+		if err != nil {
+			t.Fatalf("Creating dynamic client: %v", err)
+		}
+
+		callback(dynamic)
+	})
+}
+
+// WithNamespace creates a namespace and deletes it after the test is done.
+// It can be used within the context of a RunKubeAPITest.
+func WithNamespace(ctx context.Context, t *testing.T, callback func(namespace string)) {
+	t.Helper()
+
+	WithKubernetes(ctx, t, func(k8s kubernetes.Interface) {
+		name := fmt.Sprintf("integration-%d", time.Now().UnixNano())
+		namespace := &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+			},
+		}
+
+		if _, err := k8s.CoreV1().Namespaces().Create(namespace); err != nil {
+			t.Fatalf("Creating namespace: %v", err)
+		}
+		defer k8s.CoreV1().Namespaces().Delete(name, &metav1.DeleteOptions{})
+
+		t.Logf("With Namespace: %s", name)
+		callback(name)
+	})
 }
 
 // CancelOnSignal watches for a kill signal. If it gets one, it invokes the
diff --git a/pkg/reconciler/integration/defaults_test.go b/pkg/reconciler/integration/defaults_test.go
new file mode 100644
index 0000000..f377fe1
--- /dev/null
+++ b/pkg/reconciler/integration/defaults_test.go
@@ -0,0 +1,162 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/google/kf/pkg/kf/testutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"sigs.k8s.io/yaml"
+)
+
+// DefaultTest defines validation for defaulting webhooks. The Resource will
+// be created and all paths will be tested for equality.
+type DefaultTest struct {
+	Path              string                 `json:"-"`
+	TestEqualityPaths []string               `json:"testEquality"`
+	Resource          map[string]interface{} `json:"resource"`
+}
+
+func (dt *DefaultTest) GetUnstructured() *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: dt.Resource,
+	}
+}
+
+func (dt *DefaultTest) GetGVR() schema.GroupVersionResource {
+	obj := dt.GetUnstructured()
+	gvk := obj.GroupVersionKind()
+
+	return schema.GroupVersionResource{
+		Group:   gvk.Group,
+		Version: gvk.Version,
+
+		// We can autodiscover this in the future from the API Server like
+		// Kubernetes does, but in the meantime all resources we deal with follow
+		// this formula.
+		Resource: strings.ToLower(gvk.Kind) + "s",
+	}
+}
+
+func loadTests(t *testing.T) []*DefaultTest {
+	t.Helper()
+
+	tests, err := filepath.Glob(filepath.Join("testdata", "defaults", "*.yaml"))
+	testutil.AssertNil(t, "defaults err", err)
+
+	// Sort so the tests are deterministic as different filesystems may traverse
+	// their directories in different ways.
+	sort.Strings(tests)
+
+	var out []*DefaultTest
+
+	for _, testPath := range tests {
+		testBytes, err := ioutil.ReadFile(testPath)
+		testutil.AssertNil(t, "read error", err)
+
+		defaultTest := &DefaultTest{}
+		err = yaml.Unmarshal(testBytes, defaultTest)
+		testutil.AssertNil(t, "unmarshal error", err)
+		defaultTest.Path = testPath
+
+		out = append(out, defaultTest)
+	}
+
+	return out
+}
+
+// TestLoadTests validates that tests can be loaded from disk
+// this test will run even if integration tests are not enabled.
+func TestLoadTests(t *testing.T) {
+	loadTests(t)
+}
+
+func TestIntegration_ApplyDefaults(t *testing.T) {
+	if testutil.ShouldSkipIntegration(t) {
+		return
+	}
+
+	for _, tc := range loadTests(t) {
+		testutil.RunKubeAPITest(t, func(ctx context.Context, t *testing.T) {
+			t.Run(tc.Path, func(t *testing.T) {
+				testutil.WithNamespace(ctx, t, func(namespace string) {
+					testutil.WithDynamicClient(ctx, t, func(client dynamic.Interface) {
+
+						dyn := client.Resource(tc.GetGVR()).Namespace(namespace)
+
+						obj := tc.GetUnstructured()
+						obj.SetNamespace(namespace)
+						expected := obj.DeepCopy()
+
+						actual, err := dyn.Create(obj, metav1.CreateOptions{})
+						testutil.AssertNil(t, "creation error", err)
+
+						// Individual paths are tested because Kubernetes doesn't have a
+						// strict system about what is/isn't allowed to be updated
+						// or the structure of resources.
+						// We need to ignore things we know are going to be set by the
+						// controller like the status, metadata UUID, etc. But maybe also
+						// things in the spec. For example, types that increment a
+						// generation field.
+						for _, path := range tc.TestEqualityPaths {
+							t.Run(path, func(t *testing.T) {
+								AssertPathEqual(t, path, expected, actual)
+							})
+						}
+					})
+				})
+			})
+		})
+	}
+}
+
+func AssertPathEqual(t *testing.T, path string, expected, actual *unstructured.Unstructured) {
+	t.Helper()
+
+	path = strings.TrimPrefix(path, ".")
+	splitPath := strings.Split(path, ".")
+
+	ev, eok, eerr := unstructured.NestedFieldNoCopy(expected.UnstructuredContent(), splitPath...)
+	testutil.AssertNil(t, "expected path error", eerr)
+
+	av, aok, aerr := unstructured.NestedFieldNoCopy(actual.UnstructuredContent(), splitPath...)
+	testutil.AssertNil(t, "actual path error", aerr)
+
+	// Test ok before trying to marshal objects that might be bad.
+	testutil.AssertEqual(t, "ok", eok, aok)
+
+	// Convert the objects to JSON, this is so we can ignore weird encoding issues
+	// like int32 vs int64.
+	evJSON, eerr := json.Marshal(ev)
+	testutil.AssertNil(t, "expected JSON error", eerr)
+
+	avJSON, aerr := json.Marshal(av)
+	testutil.AssertNil(t, "actual JSON error", aerr)
+
+	if !reflect.DeepEqual(evJSON, avJSON) {
+		testutil.AssertEqual(t, "values", ev, av)
+	}
+}
diff --git a/pkg/reconciler/integration/testdata/defaults/README.md b/pkg/reconciler/integration/testdata/defaults/README.md
new file mode 100644
index 0000000..4a77516
--- /dev/null
+++ b/pkg/reconciler/integration/testdata/defaults/README.md
@@ -0,0 +1,11 @@
+This directory contains tests for `defaults_test.go`.
+
+Each file in this directory that ends with `.yaml` will be tested. The files
+must follow the structure defined by the `DefaultTest` struct.
+
+Each test is intended to validate the defaulting webhook doesn't introduce new
+defaults. This can cause reconciliation to get stuck in an infinite loop where
+a resource is created, the defaulter changes it, the reconciler notices a
+difference and perpetually attempts to refresh it.
+
+NOTE: All the tests are positive.
diff --git a/pkg/reconciler/integration/testdata/defaults/serving_v1alpha1.yaml b/pkg/reconciler/integration/testdata/defaults/serving_v1alpha1.yaml
new file mode 100644
index 0000000..1258430
--- /dev/null
+++ b/pkg/reconciler/integration/testdata/defaults/serving_v1alpha1.yaml
@@ -0,0 +1,27 @@
+# This file is used to assert that the defaulting for serving doesn't change.
+# It is a minimal subset of fields that ensures the object will be successfully
+# created and no additional defaults are added.
+testEquality:
+- ".metadata.labels"
+- ".metadata.name"
+- ".metadata.namespace"
+- ".spec"
+resource:
+  apiVersion: serving.knative.dev/v1alpha1
+  kind: Service
+  metadata:
+    name: test
+    namespace: test
+  spec:
+    template:
+      metadata:
+        creationTimestamp: null
+      spec:
+        containers:
+        - image: gcr.io/must-be-set
+          name: ""
+          resources: {}
+        timeoutSeconds: 300
+    traffic:
+    - latestRevision: true
+      percent: 100
diff --git a/pkg/reconciler/route/config/store_test.go b/pkg/reconciler/route/config/store_test.go
new file mode 100644
index 0000000..3a47ba0
--- /dev/null
+++ b/pkg/reconciler/route/config/store_test.go
@@ -0,0 +1,54 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/kf/pkg/kf/testutil"
+	logtesting "knative.dev/pkg/logging/testing"
+
+	cmtesting "knative.dev/pkg/configmap/testing"
+)
+
+func TestStoreLoadDefault(t *testing.T) {
+	store := NewDefaultConfigStore(logtesting.TestLogger(t))
+	routingConfig := FromContext(store.ToContext(context.Background())).Routing
+
+	testutil.AssertEqual(t, "ingress name", DefaultIngressServiceName, routingConfig.IngressServiceName)
+	testutil.AssertEqual(t, "ingress ns", DefaultIngressNamespace, routingConfig.IngressNamespace)
+	testutil.AssertEqual(t, "knative ingress", fmt.Sprintf("%s/%s", KnativeServingNamespace, DefaultKnativeIngressGateway), routingConfig.KnativeIngressGateway)
+	testutil.AssertEqual(t, "gateway host ", fmt.Sprintf("%s.%s.svc.cluster.local", DefaultIngressServiceName, DefaultIngressNamespace), routingConfig.GatewayHost())
+}
+
+func TestStoreLoadWithContext(t *testing.T) {
+	store := NewDefaultConfigStore(logtesting.TestLogger(t))
+	_, routingConfig := cmtesting.ConfigMapsFromTestFile(t, RoutingConfigName)
+	store.OnConfigChanged(routingConfig)
+	config := FromContext(store.ToContext(context.Background()))
+
+	expected, _ := NewRoutingConfigFromConfigMap(routingConfig)
+	if diff := cmp.Diff(expected, config.Routing); diff != "" {
+		t.Errorf("Unexpected routing config (-want, +got): %v", diff)
+	}
+
+	testutil.AssertEqual(t, "ingress name", "test-ingress-svc", expected.IngressServiceName)
+	testutil.AssertEqual(t, "ingress ns", "test-ingress-ns", expected.IngressNamespace)
+	testutil.AssertEqual(t, "knative ingress", "knative-serving/test-knative-ingress", expected.KnativeIngressGateway)
+	testutil.AssertEqual(t, "gateway host", "test-ingress-svc.test-ingress-ns.svc.cluster.local", expected.GatewayHost())
+}
diff --git a/pkg/reconciler/route/config/testdata/config-routing.yaml b/pkg/reconciler/route/config/testdata/config-routing.yaml
new file mode 100644
index 0000000..0a5cfa4
--- /dev/null
+++ b/pkg/reconciler/route/config/testdata/config-routing.yaml
@@ -0,0 +1,39 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-routing
+  namespace: kf
+data:
+  _example: |
+    ################################
+    #                              #
+    #    EXAMPLE CONFIGURATION     #
+    #                              #
+    ################################
+
+    # This block is not actually functional configuration,
+    # but serves to illustrate the available configuration
+    # options and document them in a way that is accessible
+    # to users that `kubectl edit` this config map.
+    #
+    # These sample configuration options may be copied out of
+    # this example block and unindented to be in the data block
+    # to actually change the configuration.
+  
+    ingress.servicename: "test-ingress-svc"
+    ingress.namespace: "test-ingress-ns"
+    knative.ingress.gateway: "test-knative-ingress"
diff --git a/pkg/reconciler/route/reconciler_test.go b/pkg/reconciler/route/reconciler_test.go
index 1a08199..e139e78 100644
--- a/pkg/reconciler/route/reconciler_test.go
+++ b/pkg/reconciler/route/reconciler_test.go
@@ -28,8 +28,10 @@ import (
 	"github.com/google/kf/pkg/kf/testutil"
 	"github.com/google/kf/pkg/reconciler"
 	appresources "github.com/google/kf/pkg/reconciler/app/resources"
+	"github.com/google/kf/pkg/reconciler/route/config"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	logtesting "knative.dev/pkg/logging/testing"
 )
 
 //go:generate mockgen --package=route --copyright_file ../../kf/internal/tools/option-builder/LICENSE_HEADER --destination=fake_listers.go --mock_names=RouteLister=FakeRouteLister,RouteNamespaceLister=FakeRouteNamespaceLister,RouteClaimLister=FakeRouteClaimLister,RouteClaimNamespaceLister=FakeRouteClaimNamespaceLister github.com/google/kf/pkg/client/listers/kf/v1alpha1 RouteLister,RouteClaimLister,RouteNamespaceLister,RouteClaimNamespaceLister
@@ -64,6 +66,7 @@ func TestReconciler_Reconcile_namespaceIsTerminating(t *testing.T) {
 		Base: &reconciler.Base{
 			NamespaceLister: fakeNamespaceLister,
 		},
+		configStore: config.NewDefaultConfigStore(logtesting.TestLogger(t)),
 	}
 
 	testutil.AssertNil(t, "err", r.Reconcile(context.Background(), `{"namespace":"some-namespace"}`))
@@ -171,7 +174,7 @@ func TestReconciler_Reconcile_ApplyChanges(t *testing.T) {
 					Return(nil)
 			},
 		},
-		"no claims, deleting Rotues returns not found": {
+		"no claims, deleting Routes returns not found": {
 			ExpectedErr: nil, // No need to return an error
 			Setup: func(t *testing.T, f fakes) {
 				f.frcnl.EXPECT().
@@ -199,7 +202,7 @@ func TestReconciler_Reconcile_ApplyChanges(t *testing.T) {
 					Return(apierrors.NewNotFound(v1alpha3.Resource("Route"), "Route"))
 			},
 		},
-		"no claims, deletes Rotues and VirtualServices": {
+		"no claims, deletes Routes and VirtualServices": {
 			Namespace: "some-namespace",
 			RouteSpecFields: v1alpha1.RouteSpecFields{
 				Hostname: "some-hostname",
@@ -528,10 +531,13 @@ func TestReconciler_Reconcile_ApplyChanges(t *testing.T) {
 				routeClaimLister:     fakeRouteClaimLister,
 				routeLister:          fakeRouteLister,
 				virtualServiceLister: fakeVirtualServiceLister,
+				configStore:          config.NewDefaultConfigStore(logtesting.TestLogger(t)),
 			}
 
+			ctx := r.configStore.ToContext(context.Background())
+
 			err := r.ApplyChanges(
-				context.Background(),
+				ctx,
 				tc.Namespace,
 				tc.RouteSpecFields,
 			)
diff --git a/pkg/reconciler/route/resources/virtual_service_test.go b/pkg/reconciler/route/resources/virtual_service_test.go
index 949c424..cbd3d61 100644
--- a/pkg/reconciler/route/resources/virtual_service_test.go
+++ b/pkg/reconciler/route/resources/virtual_service_test.go
@@ -15,18 +15,23 @@
 package resources_test
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"os"
 	"testing"
 
 	"github.com/google/kf/pkg/apis/kf/v1alpha1"
 	"github.com/google/kf/pkg/kf/testutil"
+	"github.com/google/kf/pkg/reconciler/route"
+	"github.com/google/kf/pkg/reconciler/route/config"
 	"github.com/google/kf/pkg/reconciler/route/resources"
 	"github.com/google/kf/third_party/knative-serving/pkg/network"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	istio "knative.dev/pkg/apis/istio/common/v1alpha1"
 	networking "knative.dev/pkg/apis/istio/v1alpha3"
+	logtesting "knative.dev/pkg/logging/testing"
 )
 
 func makeRouteSpecFields(host, domain, path string) v1alpha1.RouteSpecFields {
@@ -278,7 +283,9 @@ func TestMakeVirtualService(t *testing.T) {
 		},
 	} {
 		t.Run(tn, func(t *testing.T) {
-			s, err := resources.MakeVirtualService(tc.Claims, tc.Routes)
+			ctx := config.NewDefaultConfigStore(logtesting.TestLogger(t)).ToContext(context.Background())
+
+			s, err := resources.MakeVirtualService(ctx, tc.Claims, tc.Routes)
 			tc.Assert(t, s, err)
 		})
 	}
@@ -297,7 +304,8 @@ func ExampleMakeVirtualService_pathMatchers() {
 		makeRoute("some-host", "example.com/", "/some-path-2", "some-app-2"),
 	}
 
-	vs, err := resources.MakeVirtualService(claims, routes)
+	ctx := config.NewDefaultConfigStore(route.NewDefaultConfigLogger(os.Stdout)).ToContext(context.Background())
+	vs, err := resources.MakeVirtualService(ctx, claims, routes)
 	if err != nil {
 		panic(err)
 	}
@@ -330,7 +338,8 @@ func ExampleMakeVirtualService_weightedRoutes() {
 		makeRoute("some-host", "example.com/", "/path-b", "app-b-6"),
 	}
 
-	vs, err := resources.MakeVirtualService(claims, routes)
+	ctx := config.NewDefaultConfigStore(route.NewDefaultConfigLogger(os.Stdout)).ToContext(context.Background())
+	vs, err := resources.MakeVirtualService(ctx, claims, routes)
 	if err != nil {
 		panic(err)
 	}