Repository: emissary-ingress/emissary

Issue:
Ambassador Isolated Configuration
There needs to be a way to configure multiple Ambassadors on a single cluster independently.

Two issues here:

1. Ambassador configuration itself is cluster global, so for example, if I set `use_proxy_proto: true` on an Ambassador config then all Ambassador's pick up this change... this makes it impossible to run different configurations on the cluster such as you might want to do for testing or canary.

2. Ambassador has a cluster global understanding of mappings. 

I'm mostly concerned about (1) here. One thought I have is making Ambassador configuration happen as a filesystem file (injected likely through a configmap). 


Discussion:
I hit the first issue working on Argonath. I need to turn proxy protocol on for the next roll out, but if I turn it on using the Ambassador config mechanism then the existing prod Ambassador will be changed as well. This will cause an outage.
We have an additional driver for this: e2e test suite parallelization
So I think the biggest concern here is we can't break existing configuration by introducing a selector mechanism.

Here's what I am thinking

- Ambassador instances are "named" are now named.
- The default name is "default"

If you want to run multiple Ambassadors the config object for Ambassador gets written like this:

```yaml
---
apiVersion: ambassador/v0
kind:  Module
name:  ambassador
config:
  instanceId: other-ambassador
```

Then when you want a mapping to target a particular Ambassador instance:

```yaml
apiVersion: ambassador/v0
kind: Mapping
name: whatever
prefix: /whatever/
ambassador: other-ambassador
```

This would cause the mapping to only appear in the instance named `other-ambassador`

Resolving patch:
diff --git a/ambassador/ambassador/config.py b/ambassador/ambassador/config.py
--- a/ambassador/ambassador/config.py
+++ b/ambassador/ambassador/config.py
@@ -98,6 +98,9 @@ class Config (object):
     # __version__ won't, and we need to be consistent for comparison.
     current_semver = get_semver("current", scout_version)
 
+    # When using multiple Ambassadors in one cluster, use AMBASSADOR_ID to distinguish them.
+    ambassador_id = os.environ.get('AMBASSADOR_ID', 'default')
+
     runtime = "kubernetes" if os.environ.get('KUBERNETES_SERVICE_HOST', None) else "docker"
     namespace = os.environ.get('AMBASSADOR_NAMESPACE', 'default')
 
@@ -396,6 +399,21 @@ def process_all_objects(self):
                 # No pragma involved here; just default to the filename.
                 self.source = self.filename
 
+            # Is an ambassador_id present in this object?
+            allowed_ids = obj.get('ambassador_id', 'default')
+
+            if allowed_ids:
+                # Make sure it's a list. Yes, this is Draconian,
+                # but the jsonschema will allow only a string or a list,
+                # and guess what? Strings are iterables.
+                if type(allowed_ids) != list:
+                    allowed_ids = [ allowed_ids ]
+
+                if Config.ambassador_id not in allowed_ids:
+                    self.logger.debug("PROCESS: skip %s.%d; id %s not in %s" % 
+                                      (self.filename, self.ocount, Config.ambassador_id, allowed_ids))
+                    continue
+
             self.logger.debug("PROCESS: %s.%d => %s" % (self.filename, self.ocount, self.source))
 
             rc = self.process_object(obj)
diff --git a/end-to-end/010-multiple-ambassadors/fix_env.py b/end-to-end/010-multiple-ambassadors/fix_env.py
new file mode 100644
--- /dev/null
+++ b/end-to-end/010-multiple-ambassadors/fix_env.py
@@ -0,0 +1,50 @@
+#!python
+
+import sys
+
+import yaml
+
+import dpath
+
+id_num = sys.argv[1]
+namespace = 'test-010-%s' % id_num
+
+manifest = list(yaml.safe_load_all(open(sys.argv[2], 'r')))
+
+kinds_to_delete = {
+    "ClusterRole": True,
+    "ServiceAccount": True,
+    "ServiceAccount": True,
+    "ClusterRoleBinding": True,
+}
+
+keep = []
+
+for x in range(len(manifest)):
+    kind = dpath.util.get(manifest, "/%d/kind" % x)
+    name = dpath.util.get(manifest, "/%d/metadata/name" % x)
+
+    if kind in kinds_to_delete:
+        # print("Skipping %s %s" % (kind, name))
+        continue
+
+    # print("Adding namespace %s to %s %s" % (namespace, kind, name))
+    dpath.util.new(manifest, "/%d/metadata/namespace" % x, namespace)
+
+    if kind == 'Deployment':
+        # print("Setting environment for %s %s" % (kind, name))
+        path = "/%d/spec/template/spec/containers/0/env" % x
+        dpath.util.new(manifest, path, [
+            {
+                'name': 'AMBASSADOR_NAMESPACE',
+                'value': namespace
+            },
+            {
+                'name': 'AMBASSADOR_ID',
+                'value': 'ambassador-%s' % id_num
+            }            
+        ])
+    
+    keep.append(manifest[x])
+
+yaml.safe_dump_all(keep, open(sys.argv[3], "w"), default_flow_style=False)
diff --git a/end-to-end/checkweights.py b/end-to-end/checkweights.py
--- a/end-to-end/checkweights.py
+++ b/end-to-end/checkweights.py
@@ -31,8 +31,8 @@
 
 headers.sort()
 
-print("Headers: %s" % headers)
-print("Wanted:  %s" % wanted)
+# print("Headers: %s" % headers)
+# print("Wanted:  %s" % wanted)
 
 r = requests.get("%s/ambassador/v0/diag/?json=true" % base_url, verify=False)
 
@@ -61,7 +61,7 @@
 
 x = [ int(clusters[name]) for name in sorted(clusters.keys()) ]
 
-print("weights: %s" % x)
+# print("weights: %s" % x)
 # print("wanted:  %s" % wanted)
 
 if x != wanted:


Test patch:
diff --git a/ambassador/tests/001-broader-v0/config/mapping-evil.yaml b/ambassador/tests/001-broader-v0/config/mapping-evil.yaml
--- a/ambassador/tests/001-broader-v0/config/mapping-evil.yaml
+++ b/ambassador/tests/001-broader-v0/config/mapping-evil.yaml
@@ -8,11 +8,23 @@ apiVersion: ambassador/v0
 kind: Mapping
 name: evil_get_mapping
 prefix: /evil/
+ambassador_id:
+  - default
+  - tester
 method: GET
 service: qotm
 ---
 apiVersion: ambassador/v0
 kind: Mapping
+name: evil_get_mapping
+ambassador_id: do_not_include
+prefix: /evil-should-not-be-here/
+method: GET
+service: qotm
+---
+apiVersion: ambassador/v0
+kind: Mapping
+ambassador_id: default
 name: evil_put_mapping
 prefix: /evil/
 rewrite: /qotm/quote/
diff --git a/ambassador/tests/001-broader-v0/config/mapping-grpc.yaml b/ambassador/tests/001-broader-v0/config/mapping-grpc.yaml
--- a/ambassador/tests/001-broader-v0/config/mapping-grpc.yaml
+++ b/ambassador/tests/001-broader-v0/config/mapping-grpc.yaml
@@ -5,3 +5,4 @@ name: grpc_mapping
 prefix: /grpc/
 grpc: true
 service: "192.168.0.1:45"
+ambassador_id: [ "default", "testing" ]
\ No newline at end of file
diff --git a/ambassador/tests/001-broader-v0/gold.intermediate.json b/ambassador/tests/001-broader-v0/gold.intermediate.json
--- a/ambassador/tests/001-broader-v0/gold.intermediate.json
+++ b/ambassador/tests/001-broader-v0/gold.intermediate.json
@@ -89,10 +89,10 @@
             },
             {
                 "_referenced_by": [
-                    "mapping-evil.yaml.2"
+                    "mapping-evil.yaml.3"
                 ],
                 "_service": "qotm",
-                "_source": "mapping-evil.yaml.2",
+                "_source": "mapping-evil.yaml.3",
                 "lb_type": "round_robin",
                 "name": "cluster_http___qotm",
                 "type": "strict_dns",
@@ -618,9 +618,9 @@
                 "_method": "PUT",
                 "_precedence": 0,
                 "_referenced_by": [
-                    "mapping-evil.yaml.2"
+                    "mapping-evil.yaml.3"
                 ],
-                "_source": "mapping-evil.yaml.2",
+                "_source": "mapping-evil.yaml.3",
                 "clusters": [
                     {
                         "name": "cluster_http___qotm",
@@ -981,7 +981,7 @@
         },
         "mapping-evil.yaml": {
             "mapping-evil.yaml.1": true,
-            "mapping-evil.yaml.2": true
+            "mapping-evil.yaml.3": true
         },
         "mapping-google.yaml": {
             "mapping-google.yaml.1": true
@@ -1098,16 +1098,16 @@
             "kind": "Mapping",
             "name": "evil_get_mapping",
             "version": "ambassador/v0",
-            "yaml": "apiVersion: ambassador/v0\nkind: Mapping\nmethod: GET\nname: evil_get_mapping\nprefix: /evil/\nservice: qotm\n"
+            "yaml": "ambassador_id:\n- default\n- tester\napiVersion: ambassador/v0\nkind: Mapping\nmethod: GET\nname: evil_get_mapping\nprefix: /evil/\nservice: qotm\n"
         },
-        "mapping-evil.yaml.2": {
+        "mapping-evil.yaml.3": {
             "_source": "test source",
             "filename": "mapping-evil.yaml",
-            "index": 2,
+            "index": 3,
             "kind": "Mapping",
             "name": "evil_put_mapping",
             "version": "ambassador/v0",
-            "yaml": "apiVersion: ambassador/v0\nkind: Mapping\nmethod: PUT\nname: evil_put_mapping\nprefix: /evil/\nrewrite: /qotm/quote/\nservice: http://qotm\n"
+            "yaml": "ambassador_id: default\napiVersion: ambassador/v0\nkind: Mapping\nmethod: PUT\nname: evil_put_mapping\nprefix: /evil/\nrewrite: /qotm/quote/\nservice: http://qotm\n"
         },
         "mapping-google.yaml.1": {
             "_source": "mapping-google.yaml",
@@ -1125,7 +1125,7 @@
             "kind": "Mapping",
             "name": "grpc_mapping",
             "version": "ambassador/v0",
-            "yaml": "apiVersion: ambassador/v0\ngrpc: true\nkind: Mapping\nname: grpc_mapping\nprefix: /grpc/\nservice: 192.168.0.1:45\n"
+            "yaml": "ambassador_id:\n- default\n- testing\napiVersion: ambassador/v0\ngrpc: true\nkind: Mapping\nname: grpc_mapping\nprefix: /grpc/\nservice: 192.168.0.1:45\n"
         },
         "mapping-httpbin-shadow.yaml.1": {
             "_source": "mapping-httpbin-shadow.yaml",
diff --git a/end-to-end/004-tls-1/test.sh b/end-to-end/004-tls-1/test.sh
--- a/end-to-end/004-tls-1/test.sh
+++ b/end-to-end/004-tls-1/test.sh
@@ -34,7 +34,7 @@ APORT=$(service_port ambassador)
 DEMOTEST_POD=$(demotest_pod)
 
 BASEURL="https://${CLUSTER}:${APORT}"
-HTTPURL="http://${CLUSTER}:$(service_port ambassador 1)"
+HTTPURL="http://${CLUSTER}:$(service_port ambassador default 1)"
 
 echo "Base URL $BASEURL"
 echo "HTTP URL $HTTPURL"
diff --git a/end-to-end/010-multiple-ambassadors/test.sh b/end-to-end/010-multiple-ambassadors/test.sh
new file mode 100644
--- /dev/null
+++ b/end-to-end/010-multiple-ambassadors/test.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+HERE=$(cd $(dirname $0); pwd)
+
+cd "$HERE"
+
+ROOT=$(cd ..; pwd)
+PATH="${ROOT}:${PATH}"
+
+source ${ROOT}/utils.sh
+
+initialize_cluster
+
+kubectl cluster-info
+
+python fix_env.py 1 ../ambassador-deployment.yaml k8s/ambassador-deployment-1.yaml
+python fix_env.py 2 ../ambassador-deployment.yaml k8s/ambassador-deployment-2.yaml
+
+kubectl create namespace test-010-1
+kubectl create namespace test-010-2
+kubectl create namespace test-010-svc
+kubectl apply -f k8s/rbac.yaml
+kubectl apply -f k8s/ambassador-1.yaml
+kubectl apply -f k8s/ambassador-2.yaml
+kubectl apply -f k8s/ambassador-deployment-1.yaml
+kubectl apply -f k8s/ambassador-deployment-2.yaml
+
+kubectl run demotest --image=dwflynn/demotest:0.0.1 -- /bin/sh -c "sleep 3600"
+
+set +e +o pipefail
+
+wait_for_pods
+
+CLUSTER=$(cluster_ip)
+APORT1=$(service_port ambassador test-010-1)
+APORT2=$(service_port ambassador test-010-2)
+DEMOTEST_POD=$(demotest_pod)
+
+BASEURL1="http://${CLUSTER}:${APORT1}"
+BASEURL2="http://${CLUSTER}:${APORT2}"
+
+echo "Base 1 URL $BASEURL1"
+echo "Diag 1 URL $BASEURL1/ambassador/v0/diag/"
+echo "Base 2 URL $BASEURL2"
+echo "Diag 2 URL $BASEURL2/ambassador/v0/diag/"
+
+wait_for_ready "$BASEURL1"
+wait_for_ready "$BASEURL2"
+
+if ! check_diag "$BASEURL1" 1-1 "No annotated services"; then
+    exit 1
+fi
+
+if ! check_diag "$BASEURL2" 1-2 "No annotated services"; then
+    exit 1
+fi
+
+kubectl apply -f k8s/demo-1.yaml
+kubectl apply -f k8s/demo-2.yaml
+
+wait_for_pods
+
+wait_for_demo_weights "$BASEURL1" 100
+wait_for_demo_weights "$BASEURL2" 100
+
+
+if ! check_diag "$BASEURL1" 2-1 "demo annotated"; then
+    exit 1
+fi
+
+if ! check_diag "$BASEURL2" 2-2 "demo annotated"; then
+    exit 1
+fi
+
+if ! kubectl exec -i $DEMOTEST_POD -- python3 demotest.py "$BASEURL1" /dev/fd/0 < demo-1.yaml; then
+    exit 1
+fi
+
+if ! kubectl exec -i $DEMOTEST_POD -- python3 demotest.py "$BASEURL2" /dev/fd/0 < demo-2.yaml; then
+    exit 1
+fi
+
+# kubernaut discard