Repository: grafana/tanka
Pull request #237 (2020-03-16 17:37:06)

fix(kubernetes): properly handle diff namespaces
Supersedes https://github.com/grafana/tanka/pull/224 with a approach that I consider more proper:

1. Cluster-wide resources are always checked with the running cluster
2. If a namespace is missing but will be created in the same run, report is as created
3. Everything else is checked with the running cluster.

This is probably a good way because:
- Fully absent namespaces are still noticed at apply
- The common case of namespace missing, but soon existing is covered

In the future, this can be easily extended to handle CRD's as well, because the list of known resources is already queried from the cluster (thanks @mplzik for the idea).


Patch:
diff --git a/docs/docs/env-vars.md b/docs/docs/env-vars.md
index 550c8f11..d145f8d1 100644
--- a/docs/docs/env-vars.md
+++ b/docs/docs/env-vars.md
@@ -9,3 +9,8 @@ route: "/env-vars"
 
 **Description**: Path to the `kubectl` tool executable  
 **Default**: `$PATH/kubectl`
+
+### TANKA_KUBECTL_TRACE
+
+**Description**: Print all calls to `kubectl`  
+**Default**: `false`
diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go
index 2f560c9d..c4d7bdc0 100644
--- a/pkg/kubernetes/client/client.go
+++ b/pkg/kubernetes/client/client.go
@@ -24,6 +24,8 @@ type Client interface {
 
 	// Namespaces the cluster currently has
 	Namespaces() (map[string]bool, error)
+	// Resources returns all known api-resources of the cluster
+	Resources() (Resources, error)
 
 	// Info returns known informational data about the client. Best effort based,
 	// fields of `Info` that cannot be stocked with valuable data, e.g.
diff --git a/pkg/kubernetes/client/diff.go b/pkg/kubernetes/client/diff.go
index 29abc35d..22aefbce 100644
--- a/pkg/kubernetes/client/diff.go
+++ b/pkg/kubernetes/client/diff.go
@@ -9,18 +9,11 @@ import (
 	"github.com/Masterminds/semver"
 
 	"github.com/grafana/tanka/pkg/kubernetes/manifest"
-	"github.com/grafana/tanka/pkg/kubernetes/util"
 )
 
 // DiffServerSide takes the desired state and computes the differences on the
 // server, returning them in `diff(1)` format
 func (k Kubectl) DiffServerSide(data manifest.List) (*string, error) {
-	ns, err := k.Namespaces()
-	if err != nil {
-		return nil, err
-	}
-
-	ready, missing := separateMissingNamespace(data, ns)
 	cmd := k.ctl("diff", "-f", "-")
 
 	raw := bytes.Buffer{}
@@ -29,28 +22,19 @@ func (k Kubectl) DiffServerSide(data manifest.List) (*string, error) {
 	fw := FilterWriter{filters: []*regexp.Regexp{regexp.MustCompile(`exit status \d`)}}
 	cmd.Stderr = &fw
 
-	cmd.Stdin = strings.NewReader(ready.String())
+	cmd.Stdin = strings.NewReader(data.String())
 
-	err = cmd.Run()
+	err := cmd.Run()
 	if diffErr := parseDiffErr(err, fw.buf, k.Info().ClientVersion); diffErr != nil {
 		return nil, diffErr
 	}
 
 	s := raw.String()
-	for _, r := range missing {
-		d, err := util.DiffStr(util.DiffName(r), "", r.String())
-		if err != nil {
-			return nil, err
-		}
-		s += d
-	}
-
-	if s != "" {
-		return &s, nil
+	if s == "" {
+		return nil, nil
 	}
 
-	// no diff -> nil
-	return nil, nil
+	return &s, nil
 }
 
 // parseDiffErr handles the exit status code of `kubectl diff`. It returns err
@@ -85,15 +69,3 @@ func parseDiffErr(err error, stderr string, version *semver.Version) error {
 	// differences found is not an error
 	return nil
 }
-
-func separateMissingNamespace(in manifest.List, exists map[string]bool) (ready, missingNamespace manifest.List) {
-	for _, r := range in {
-		// namespace does not exist, also ignore implicit default ("")
-		if ns := r.Metadata().Namespace(); ns != "" && !exists[ns] {
-			missingNamespace = append(missingNamespace, r)
-			continue
-		}
-		ready = append(ready, r)
-	}
-	return
-}
diff --git a/pkg/kubernetes/client/errors.go b/pkg/kubernetes/client/errors.go
index 8d25b96e..356f7831 100644
--- a/pkg/kubernetes/client/errors.go
+++ b/pkg/kubernetes/client/errors.go
@@ -25,12 +25,12 @@ func (e ErrorUnknownResource) Error() string {
 type ErrorNoContext string
 
 func (e ErrorNoContext) Error() string {
-	return fmt.Sprintf("no context named `%s` was found. Please check your $KUBECONFIG", e)
+	return fmt.Sprintf("no context named `%s` was found. Please check your $KUBECONFIG", string(e))
 }
 
 // ErrorNoCluster means that the cluster that was searched for couldn't be found
 type ErrorNoCluster string
 
 func (e ErrorNoCluster) Error() string {
-	return fmt.Sprintf("no cluster that matches the apiServer `%s` was found. Please check your $KUBECONFIG", e)
+	return fmt.Sprintf("no cluster that matches the apiServer `%s` was found. Please check your $KUBECONFIG", string(e))
 }
diff --git a/pkg/kubernetes/client/exec.go b/pkg/kubernetes/client/exec.go
index b4c5e12c..34ceb76c 100644
--- a/pkg/kubernetes/client/exec.go
+++ b/pkg/kubernetes/client/exec.go
@@ -32,6 +32,10 @@ func (k Kubectl) ctl(action string, args ...string) *exec.Cmd {
 	cmd := kubectlCmd(argv...)
 	cmd.Env = patchKubeconfig(k.nsPatch, os.Environ())
 
+	if os.Getenv("TANKA_KUBECTL_TRACE") != "" {
+		fmt.Println(cmd.String())
+	}
+
 	return cmd
 }
 
diff --git a/pkg/kubernetes/client/resources.go b/pkg/kubernetes/client/resources.go
new file mode 100644
index 00000000..6a30c1a0
--- /dev/null
+++ b/pkg/kubernetes/client/resources.go
@@ -0,0 +1,128 @@
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/grafana/tanka/pkg/kubernetes/manifest"
+)
+
+// Resources the Kubernetes API knows
+type Resources []Resource
+
+// Namespaced returns whether a resource is namespace-specific or cluster-wide
+func (r Resources) Namespaced(m manifest.Manifest) bool {
+	for _, res := range r {
+		if m.Kind() == res.Kind {
+			return res.Namespaced
+		}
+	}
+
+	return false
+}
+
+// Resource is a Kubernetes API Resource
+type Resource struct {
+	APIGroup   string `json:"APIGROUP"`
+	Kind       string `json:"KIND"`
+	Name       string `json:"NAME"`
+	Namespaced bool   `json:"NAMESPACED,string"`
+	Shortnames string `json:"SHORTNAMES"`
+}
+
+// Resources returns all API resources known to the server
+func (k Kubectl) Resources() (Resources, error) {
+	cmd := k.ctl("api-resources", "--cached")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		return nil, err
+	}
+
+	var res Resources
+	if err := UnmarshalTable(out.String(), &res); err != nil {
+		return nil, errors.Wrap(err, "parsing table")
+	}
+
+	return res, nil
+}
+
+// UnmarshalTable unmarshals a raw CLI table into ptr. `json` package is used
+// for getting the dat into the ptr, `json:` struct tags can be used.
+func UnmarshalTable(raw string, ptr interface{}) error {
+	raw = strings.TrimSpace(raw)
+
+	lines := strings.Split(raw, "\n")
+	if len(lines) == 0 {
+		return ErrorNoHeader
+	}
+
+	headerStr := lines[0]
+	// headers are ALL-CAPS
+	if !regexp.MustCompile(`^[A-Z\s]+$`).MatchString(headerStr) {
+		return ErrorNoHeader
+	}
+
+	lines = lines[1:]
+
+	spc := regexp.MustCompile(`[A-Z]+\s*`)
+	header := spc.FindAllString(headerStr, -1)
+
+	tbl := make([]map[string]string, 0, len(lines))
+	for _, l := range lines {
+		elems := splitRow(l, header)
+		if len(elems) != len(header) {
+			return ErrorElementsMismatch{Header: len(header), Row: len(elems)}
+		}
+
+		row := make(map[string]string)
+		for i, e := range elems {
+			key := strings.TrimSpace(header[i])
+			row[key] = strings.TrimSpace(e)
+		}
+		tbl = append(tbl, row)
+	}
+
+	j, err := json.Marshal(tbl)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(j, ptr)
+}
+
+// ErrorNoHeader occurs when the table lacks an ALL-CAPS header
+var ErrorNoHeader = fmt.Errorf("table has no header")
+
+// ErrorElementsMismatch occurs when a row has a different count of elements
+// than it's header
+type ErrorElementsMismatch struct {
+	Header, Row int
+}
+
+func (e ErrorElementsMismatch) Error() string {
+	return fmt.Sprintf("header and row have different element count: %v != %v", e.Header, e.Row)
+}
+
+func splitRow(s string, header []string) (elems []string) {
+	pos := 0
+	for i, h := range header {
+		if i == len(header)-1 {
+			elems = append(elems, s[pos:])
+			continue
+		}
+
+		lim := len(h)
+		elems = append(elems, s[pos:pos+lim])
+		pos += lim
+	}
+	return elems
+}
diff --git a/pkg/kubernetes/diff.go b/pkg/kubernetes/diff.go
new file mode 100644
index 00000000..ffe01e99
--- /dev/null
+++ b/pkg/kubernetes/diff.go
@@ -0,0 +1,188 @@
+package kubernetes
+
+import (
+	"fmt"
+
+	"github.com/grafana/tanka/pkg/kubernetes/client"
+	"github.com/grafana/tanka/pkg/kubernetes/manifest"
+	"github.com/grafana/tanka/pkg/kubernetes/util"
+	"github.com/pkg/errors"
+)
+
+// Diff takes the desired state and returns the differences from the cluster
+func (k *Kubernetes) Diff(state manifest.List, opts DiffOpts) (*string, error) {
+	// required for separating
+	namespaces, err := k.ctl.Namespaces()
+	if err != nil {
+		return nil, errors.Wrap(err, "listing namespaces")
+	}
+	resources, err := k.ctl.Resources()
+	if err != nil {
+		return nil, errors.Wrap(err, "listing known api-resources")
+	}
+
+	// separate resources in groups
+	//
+	// soon: resources that have unmet dependencies that will be met during
+	// apply. These will be diffed statically, because checking with the cluster
+	// would cause an error
+	//
+	// live: all other resources
+	live, soon := separate(state, k.Spec.Namespace, separateOpts{
+		namespaces: namespaces,
+		resources:  resources,
+	})
+
+	// differ for live resources
+	liveDiff, err := k.differ(opts.Strategy)
+	if err != nil {
+		return nil, err
+	}
+
+	// reports all resources as new
+	staticDiff := StaticDiffer(true)
+
+	// run the diff
+	d, err := multiDiff{
+		{differ: liveDiff, state: live},
+		{differ: staticDiff, state: soon},
+	}.diff()
+
+	switch {
+	case err != nil:
+		return nil, err
+	case d == nil:
+		return nil, nil
+	}
+
+	if opts.Summarize {
+		return util.Diffstat(*d)
+	}
+
+	return d, nil
+}
+
+type separateOpts struct {
+	namespaces map[string]bool
+	resources  client.Resources
+}
+
+func separate(state manifest.List, defaultNs string, opts separateOpts) (live manifest.List, soon manifest.List) {
+	soonNs := make(map[string]bool)
+	for _, m := range state {
+		if m.Kind() != "Namespace" {
+			continue
+		}
+		soonNs[m.Metadata().Name()] = true
+	}
+
+	for _, m := range state {
+		// non-namespaced always live
+		if !opts.resources.Namespaced(m) {
+			live = append(live, m)
+			continue
+		}
+
+		// handle implicit default
+		ns := m.Metadata().Namespace()
+		if ns == "" {
+			ns = defaultNs
+		}
+
+		// special case: namespace missing, BUT will be created during apply
+		if !opts.namespaces[ns] && soonNs[ns] {
+			soon = append(soon, m)
+			continue
+		}
+
+		// everything else
+		live = append(live, m)
+	}
+
+	return live, soon
+}
+
+// ErrorDiffStrategyUnknown occurs when a diff-strategy is requested that does
+// not exist.
+type ErrorDiffStrategyUnknown struct {
+	Requested string
+	differs   map[string]Differ
+}
+
+func (e ErrorDiffStrategyUnknown) Error() string {
+	strats := []string{}
+	for s := range e.differs {
+		strats = append(strats, s)
+	}
+	return fmt.Sprintf("diff strategy `%s` does not exist. Pick one of: %v", e.Requested, strats)
+}
+
+func (k *Kubernetes) differ(override string) (Differ, error) {
+	strategy := k.Spec.DiffStrategy
+	if override != "" {
+		strategy = override
+	}
+
+	d, ok := k.differs[strategy]
+	if !ok {
+		return nil, ErrorDiffStrategyUnknown{
+			Requested: strategy,
+			differs:   k.differs,
+		}
+	}
+
+	return d, nil
+}
+
+// StaticDiffer returns a differ that reports all resources as either created or
+// deleted.
+func StaticDiffer(create bool) Differ {
+	return func(state manifest.List) (*string, error) {
+		s := ""
+		for _, m := range state {
+			is, should := m.String(), ""
+			if create {
+				is, should = should, is
+			}
+
+			d, err := util.DiffStr(util.DiffName(m), is, should)
+			if err != nil {
+				return nil, err
+			}
+			s += d
+		}
+
+		if s == "" {
+			return nil, nil
+		}
+
+		return &s, nil
+	}
+}
+
+// multiDiff runs multiple differs (in series). In the future it might be worth
+// parallelizing this.
+type multiDiff []struct {
+	differ Differ
+	state  manifest.List
+}
+
+func (m multiDiff) diff() (*string, error) {
+	diff := ""
+	for _, d := range m {
+		s, err := d.differ(d.state)
+		if err != nil {
+			return nil, err
+		}
+
+		if s == nil {
+			continue
+		}
+		diff += *s
+	}
+
+	if diff == "" {
+		return nil, nil
+	}
+	return &diff, nil
+}
diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go
index a343150a..ad0c01c7 100644
--- a/pkg/kubernetes/kubernetes.go
+++ b/pkg/kubernetes/kubernetes.go
@@ -4,11 +4,9 @@ import (
 	"fmt"
 
 	"github.com/Masterminds/semver"
-	"github.com/pkg/errors"
 
 	"github.com/grafana/tanka/pkg/kubernetes/client"
 	"github.com/grafana/tanka/pkg/kubernetes/manifest"
-	"github.com/grafana/tanka/pkg/kubernetes/util"
 	"github.com/grafana/tanka/pkg/spec/v1alpha1"
 )
 
@@ -32,7 +30,7 @@ func New(s v1alpha1.Spec) (*Kubernetes, error) {
 	// setup client
 	ctl, err := client.New(s.APIServer, s.Namespace)
 	if err != nil {
-		return nil, errors.Wrap(err, "creating client")
+		return nil, err
 	}
 
 	// setup diffing
@@ -70,28 +68,6 @@ type DiffOpts struct {
 	Strategy string
 }
 
-// Diff takes the desired state and returns the differences from the cluster
-func (k *Kubernetes) Diff(state manifest.List, opts DiffOpts) (*string, error) {
-	strategy := k.Spec.DiffStrategy
-	if opts.Strategy != "" {
-		strategy = opts.Strategy
-	}
-
-	d, err := k.differs[strategy](state)
-	switch {
-	case err != nil:
-		return nil, err
-	case d == nil:
-		return nil, nil
-	}
-
-	if opts.Summarize {
-		return util.Diffstat(*d)
-	}
-
-	return d, nil
-}
-
 // Info about the client, etc.
 func (k *Kubernetes) Info() client.Info {
 	return k.ctl.Info()


Test patch:
diff --git a/pkg/kubernetes/client/diff_test.go b/pkg/kubernetes/client/diff_test.go
deleted file mode 100644
index 2b3dfa8d..00000000
--- a/pkg/kubernetes/client/diff_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package client
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/grafana/tanka/pkg/kubernetes/manifest"
-)
-
-func TestSeparateMissingNamespace(t *testing.T) {
-	cases := []struct {
-		name string
-		td   nsTd
-
-		missing bool
-	}{
-		// default should always exist
-		{
-			name: "default",
-			td: newNsTd(func(m manifest.Metadata) {
-				m["namespace"] = "default"
-			}, []string{}),
-			missing: false,
-		},
-		// implcit default (not specfiying an ns at all) also
-		{
-			name: "implicit-default",
-			td: newNsTd(func(m manifest.Metadata) {
-				delete(m, "namespace")
-			}, []string{}),
-			missing: false,
-		},
-		// custom ns that exists
-		{
-			name: "custom-ns",
-			td: newNsTd(func(m manifest.Metadata) {
-				m["namespace"] = "custom"
-			}, []string{"custom"}),
-			missing: false,
-		},
-		// custom ns that does not exist
-		{
-			name: "missing-ns",
-			td: newNsTd(func(m manifest.Metadata) {
-				m["namespace"] = "missing"
-			}, []string{}),
-			missing: true,
-		},
-	}
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			ready, missing := separateMissingNamespace(manifest.List{c.td.m}, c.td.ns)
-			if c.missing {
-				assert.Lenf(t, ready, 0, "expected manifest to be missing (ready = 0)")
-				assert.Lenf(t, missing, 1, "expected manifest to be missing (missing = 1)")
-			} else {
-				assert.Lenf(t, ready, 1, "expected manifest to be ready (ready = 1)")
-				assert.Lenf(t, missing, 0, "expected manifest to be ready (missing = 0)")
-			}
-		})
-	}
-}
-
-type nsTd struct {
-	m  manifest.Manifest
-	ns map[string]bool
-}
-
-func newNsTd(f func(m manifest.Metadata), ns []string) nsTd {
-	m := manifest.Manifest{
-		"apiVersion": "apps/v1",
-		"kind":       "Deployment",
-		"metadata":   map[string]interface{}{},
-	}
-	if f != nil {
-		f(m.Metadata())
-	}
-
-	nsMap := map[string]bool{
-		"default": true, // you can't get rid of this one ever
-	}
-	for _, n := range ns {
-		nsMap[n] = true
-	}
-
-	return nsTd{
-		m:  m,
-		ns: nsMap,
-	}
-}
diff --git a/pkg/kubernetes/client/resources_test.go b/pkg/kubernetes/client/resources_test.go
new file mode 100644
index 00000000..137e08f3
--- /dev/null
+++ b/pkg/kubernetes/client/resources_test.go
@@ -0,0 +1,76 @@
+package client
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalTable(t *testing.T) {
+	cases := []struct {
+		name string
+		tbl  string
+		dest interface{}
+		want interface{}
+		err  error
+	}{
+		{
+			name: "normal",
+			tbl:  strings.TrimSpace(tblNormal),
+			want: &Resources{
+				{APIGroup: "apps", Name: "Deployment", Namespaced: true},
+				{APIGroup: "networking", Name: "Ingress", Namespaced: true},
+				{APIGroup: "", Name: "Namespace", Namespaced: false},
+				{APIGroup: "extensions", Name: "DaemonSet", Namespaced: true},
+			},
+			dest: &Resources{},
+		},
+		{
+			name: "empty",
+			tbl:  strings.TrimSpace(tblEmpty),
+			want: &Resources{},
+			dest: &Resources{
+				{APIGroup: "apps", Name: "Deployment", Namespaced: true},
+			},
+		},
+		{
+			name: "no-header",
+			tbl:  strings.TrimSpace(tblNoHeader),
+			err:  ErrorNoHeader,
+		},
+		{
+			name: "nothing",
+			tbl:  tblNothing,
+			err:  ErrorNoHeader,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			err := UnmarshalTable(c.tbl, c.dest)
+			require.Equal(t, c.err, err)
+			assert.Equal(t, c.want, c.dest)
+		})
+	}
+}
+
+var tblNormal = `
+APIGROUP    NAME        NAMESPACED
+apps        Deployment  true
+networking  Ingress     true
+            Namespace   false
+extensions  DaemonSet   true
+`
+
+var tblEmpty = `
+APIGROUP    NAME        NAMESPACED
+`
+
+var tblNoHeader = `
+apps        Deployment  true
+networking  Ingress     true
+`
+
+var tblNothing = ``
diff --git a/pkg/kubernetes/diff_test.go b/pkg/kubernetes/diff_test.go
new file mode 100644
index 00000000..2151dd88
--- /dev/null
+++ b/pkg/kubernetes/diff_test.go
@@ -0,0 +1,146 @@
+package kubernetes
+
+import (
+	"testing"
+
+	"github.com/grafana/tanka/pkg/kubernetes/client"
+	"github.com/grafana/tanka/pkg/kubernetes/manifest"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestSeparate checks that separate properly separates resources:
+//
+// - cluster-wide resources are always `live`
+// - resources with missing namespaces:
+//   - `soon` if their namespace is included in the Jsonnet
+//   - `live` otherwise, to cause a helpful error message
+//      for the user that the namespace is indeed missing
+// - `namespace: ""` should be properly translated into the default namespace
+func TestSeparate(t *testing.T) {
+	cases := []struct {
+		name string
+
+		// state returned from Jsonnet
+		state manifest.List
+		// namespaces that exist
+		namespaces []string
+
+		// the default namespace (no value -> implicit default '')
+		defaultNs string
+
+		// resources that can be checked with the cluster
+		live manifest.List
+		// resources depending on a condition that will be met on next apply
+		soon manifest.List
+	}{
+		{
+			name:      "multi",
+			defaultNs: "default",
+			namespaces: []string{
+				"default",
+				"kube-system",
+				"custom",
+			},
+			state: manifest.List{
+				// cluster-wide resources: always live
+				m("rbac.authorization.k8s.io/v1", "ClusterRole", "globalRole", ""),
+				m("rbac.authorization.k8s.io/v1", "ClusterRoleBinding", "binding", "whydoihaveanamespace"),
+
+				// default, existing namespace: `live`
+				m("apps/v1", "Deployment", "loki", ""),
+				m("apps/v1", "Deployment", "grafana", "default"),
+
+				// custom, existing namespace: `live`
+				m("apps/v1", "Deployment", "cortex", "custom"),
+
+				// custom, soon existing namespace:
+				m("v1", "Namespace", "monitoring", ""),                 // `live`
+				m("apps/v1", "Deployment", "prometheus", "monitoring"), // `soon`
+
+				// custom, missing namespace: `live`
+				m("apps/v1", "Deployment", "metrictank", "metrics"),
+			},
+			live: manifest.List{
+				m("rbac.authorization.k8s.io/v1", "ClusterRole", "globalRole", ""),
+				m("rbac.authorization.k8s.io/v1", "ClusterRoleBinding", "binding", "whydoihaveanamespace"),
+				m("apps/v1", "Deployment", "loki", ""),
+				m("apps/v1", "Deployment", "grafana", "default"),
+				m("apps/v1", "Deployment", "cortex", "custom"),
+				m("v1", "Namespace", "monitoring", ""),
+				m("apps/v1", "Deployment", "metrictank", "metrics"),
+			},
+			soon: manifest.List{
+				m("apps/v1", "Deployment", "prometheus", "monitoring"),
+			},
+		},
+		{
+			name:       "default/soon",
+			defaultNs:  "grafana",
+			namespaces: []string{"default", "kube-system"}, // `grafana` missing
+			state: manifest.List{
+				m("v1", "Namespace", "grafana", ""), // `grafana` created during apply
+				m("apps/v1", "Deployment", "prometheus", "grafana"),
+				m("apps/v1", "Deployment", "cortex", ""), // implicit default `""`
+			},
+			live: manifest.List{
+				m("v1", "Namespace", "grafana", ""),
+			},
+			soon: manifest.List{
+				m("apps/v1", "Deployment", "prometheus", "grafana"),
+				m("apps/v1", "Deployment", "cortex", ""),
+			},
+		},
+		{
+			name:       "default/missing",
+			defaultNs:  "grafana",
+			namespaces: []string{"default", "kube-system"}, // `grafana` missing
+			state: manifest.List{
+				// m("", "Namespace", "grafana", ""), <- `grafana` NOT created
+				m("apps/v1", "Deployment", "prometheus", "grafana"),
+				m("apps/v1", "Deployment", "cortex", ""), // implicit default (`""`)
+			},
+			live: manifest.List{
+				// `live`, so user notices missing ns
+				m("apps/v1", "Deployment", "prometheus", "grafana"),
+				m("apps/v1", "Deployment", "cortex", ""),
+			},
+		},
+	}
+
+	// static set of resources for this test (usually obtained using
+	// `client.Resources()`)
+	staticResources := client.Resources{
+		{APIGroup: "", Kind: "Namespace", Namespaced: false},
+		{APIGroup: "apps/v1", Kind: "Deployment", Namespaced: true},
+		{APIGroup: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole", Namespaced: false},
+		{APIGroup: "rbac.authorization.k8s.io/v1", Kind: "ClusterRoleBinding", Namespaced: false},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			namespaces := make(map[string]bool)
+			for _, n := range c.namespaces {
+				namespaces[n] = true
+			}
+
+			live, soon := separate(c.state, c.defaultNs, separateOpts{
+				namespaces: namespaces,
+				resources:  staticResources,
+			})
+
+			assert.ElementsMatch(t, c.live, live, "live")
+			assert.ElementsMatch(t, c.soon, soon, "soon")
+		})
+	}
+}
+
+func m(apiVersion, kind, name, namespace string) manifest.Manifest {
+	return manifest.Manifest{
+		"apiVersion": apiVersion,
+		"kind":       kind,
+		"metadata": map[string]interface{}{
+			"name":      name,
+			"namespace": namespace,
+		},
+	}
+}