Repository: emissary-ingress/emissary

Issue:
Ambassador potentially is a bit to eager to report uptime / heartbeats to scout.
A single deployment of Ambassador under load (Kubecon demo) reported to scout over 150,000 times... which seems excessively high. We need to investigate if this is expected behavior. 


Discussion:

Resolving patch:
diff --git a/ambassador/ambassador/config.py b/ambassador/ambassador/config.py
--- a/ambassador/ambassador/config.py
+++ b/ambassador/ambassador/config.py
@@ -1,6 +1,7 @@
 import sys
 
 import collections
+import datetime
 import json
 import logging
 import os
@@ -89,20 +90,49 @@ class Config (object):
     scout_latest_semver = None
     scout_notices = []
 
+    scout_last_response = None
+    scout_last_update = datetime.datetime.now() - datetime.timedelta(hours=24)
+    scout_update_frequency = datetime.timedelta(hours=4)
+
     @classmethod
     def scout_report(klass, force_result=None, **kwargs):
+        _notices = []
+
+        env_result = os.environ.get("AMBASSADOR_SCOUT_RESULT", None)
+        if env_result:
+            force_result = json.loads(env_result)
+
         result = force_result
+        result_timestamp = None
+        result_was_cached = False
 
         if not result:
             if Config.scout:
                 if 'runtime' not in kwargs:
                     kwargs['runtime'] = Config.runtime
 
-                result = Config.scout.report(**kwargs)
+                # How long since the last Scout update? If it's been more than an hour, 
+                # check Scout again.
+
+                now = datetime.datetime.now()
+
+                if (now - Config.scout_last_update) > Config.scout_update_frequency:
+                    result = Config.scout.report(**kwargs)
+
+                    Config.scout_last_update = now
+                    Config.scout_last_result = dict(**result)
+                else:
+                    # _notices.append({ "level": "debug", "message": "Returning cached result" })
+                    result = dict(**Config.scout_last_result)
+                    result_was_cached = True
+
+                result_timestamp = Config.scout_last_update
             else:
                 result = { "scout": "unavailable" }
-
-        _notices = []
+                result_timestamp = datetime.datetime.now()
+        else:
+            _notices.append({ "level": "debug", "message": "Returning forced result" })
+            result_timestamp = datetime.datetime.now()
 
         if not Config.current_semver:
             _notices.append({
@@ -110,6 +140,9 @@ def scout_report(klass, force_result=None, **kwargs):
                 "message": "Ambassador has bad version '%s'??!" % Config.scout_version
             })
 
+        result['cached'] = result_was_cached
+        result['timestamp'] = result_timestamp.timestamp()
+
         # Do version & notices stuff.      
         if 'latest_version' in result:
             latest_version = result['latest_version']
@@ -1043,8 +1076,8 @@ def diagnostic_overview(self):
         source_files = {}
     
         for filename, source_keys in self.source_map.items():
-            self.logger.debug("overview -- filename %s, source_keys %d" %
-                              (filename, len(source_keys)))
+            # self.logger.debug("overview -- filename %s, source_keys %d" %
+            #                   (filename, len(source_keys)))
 
             # Skip '--internal--' etc.
             if filename.startswith('--'):
@@ -1063,7 +1096,7 @@ def diagnostic_overview(self):
             )
 
             for source_key in source_keys:
-                self.logger.debug("overview --- source_key %s" % source_key)
+                # self.logger.debug("overview --- source_key %s" % source_key)
 
                 source = self.sources[source_key]
                 raw_errors = self.errors.get(source_key, [])
@@ -1106,7 +1139,7 @@ def diagnostic_overview(self):
                         routes=routes,
                         **configuration)
 
-        self.logger.debug("overview result %s" % json.dumps(overview, indent=4, sort_keys=True))
+        # self.logger.debug("overview result %s" % json.dumps(overview, indent=4, sort_keys=True))
 
         return overview
 
diff --git a/ambassador/ambassador_diag/diagd.py b/ambassador/ambassador_diag/diagd.py
--- a/ambassador/ambassador_diag/diagd.py
+++ b/ambassador/ambassador_diag/diagd.py
@@ -27,7 +27,6 @@
 __version__ = Version
 
 boot_time = datetime.datetime.now()
-last_scout_update = datetime.datetime.now() - datetime.timedelta(hours=24)
 
 logging.basicConfig(
     level=logging.INFO,
@@ -114,20 +113,14 @@ def aconf(app):
 
     aconf = Config(latest)
 
-    # How long since the last Scout update? If it's been more than an hour, 
-    # check Scout again.
+    uptime = datetime.datetime.now() - boot_time
+    hr_uptime = td_format(uptime)
 
-    now = datetime.datetime.now()
+    result = Config.scout_report(mode="diagd", runtime=Config.runtime,
+                                 uptime=int(uptime.total_seconds()),
+                                 hr_uptime=hr_uptime)
 
-    if (now - last_scout_update) > datetime.timedelta(hours=1):
-        uptime = now - boot_time
-        hr_uptime = td_format(uptime)
-
-        result = Config.scout_report(mode="diagd", runtime=Config.runtime,
-                                     uptime=int(uptime.total_seconds()),
-                                     hr_uptime=hr_uptime)
-
-        app.logger.debug("Scout reports %s" % json.dumps(result))
+    app.logger.info("Scout reports %s" % json.dumps(result))
 
     return aconf
 
diff --git a/ambassador/kubewatch.py b/ambassador/kubewatch.py
--- a/ambassador/kubewatch.py
+++ b/ambassador/kubewatch.py
@@ -3,19 +3,33 @@
 import binascii
 import click
 import json
+import logging
 import os
 import shutil
 import signal
 import subprocess
 import threading
 import time
-import traceback
 
 import yaml
 
 from kubernetes import client, config, watch
 from ambassador.config import Config
 
+from ambassador.VERSION import Version
+
+__version__ = Version
+
+logging.basicConfig(
+    level=logging.INFO, # if appDebug else logging.INFO,
+    format="%%(asctime)s kubewatch %s %%(levelname)s: %%(message)s" % __version__,
+    datefmt="%Y-%m-%d %H:%M:%S"
+)
+
+# logging.getLogger("datawire.scout").setLevel(logging.DEBUG)
+logger = logging.getLogger("kubewatch")
+logger.setLevel(logging.DEBUG)
+
 KEY = "getambassador.io/config"
 
 def is_annotated(svc):
@@ -64,14 +78,14 @@ def __init__(self, ambassador_config_dir, namespace, envoy_config_file, delay, p
 
     def read_fs(self, path):
         if os.path.exists(path):
-            print("Merging config inputs from %s" % path)
+            logger.info("Merging config inputs from %s" % path)
 
             for name in os.listdir(path):
                 if name.endswith(".yaml"):
                     with open(os.path.join(path, name)) as fd:
                         self.configs[name] = fd.read()
 
-                    print ("Loaded %s" % os.path.join(path, name))
+                    logger.info("Loaded %s" % os.path.join(path, name))
 
     def changes(self):
         with self.mutex:
@@ -85,11 +99,12 @@ def run(self):
             with self.mutex:
                 changes = self.changes()
                 if changes > 0:
-                    print("Processing %s changes" % (changes))
+                    logger.info("Processing %s changes" % (changes))
                     try:
                         self.restart()
                     except:
-                        traceback.print_exc()
+                        logging.exception("could not restart Envoy")
+
                     self.processed += changes
 
     def restart(self):
@@ -99,7 +114,7 @@ def restart(self):
         base, ext = os.path.splitext(self.envoy_config_file)
         target = "%s-%s%s" % (base, self.restart_count, ext)
         os.rename(config, target)
-        print ("Moved valid configuration %s to %s" % (config, target))
+        logger.info("Moved valid configuration %s to %s" % (config, target))
         if self.pid:
             os.kill(self.pid, signal.SIGHUP)
 
@@ -111,12 +126,15 @@ def generate_config(self, output):
             path = os.path.join(output, filename)
             with open(path, "w") as fd:
                 fd.write(config)
-            print ("Wrote %s to %s" % (filename, path))
+            logger.info("Wrote %s to %s" % (filename, path))
+
+        logger.info("generating config with gencount %d" % self.restart_count)
 
         aconf = Config(output)
-        rc = aconf.generate_envoy_config(mode="kubewatch")
+        rc = aconf.generate_envoy_config(mode="kubewatch",
+                                         generation_count=self.restart_count)
 
-        print("Scout reports %s" % json.dumps(rc.scout_result))       
+        logger.info("Scout reports %s" % json.dumps(rc.scout_result))       
 
         if rc:
             envoy_config = "%s-%s" % (output, "envoy.json")
@@ -125,16 +143,16 @@ def generate_config(self, output):
                 result = subprocess.check_output(["/usr/local/bin/envoy", "--base-id", "1", "--mode", "validate",
                                                   "-c", envoy_config])
                 if result.strip().endswith(b" OK"):
-                    print ("Configuration %s valid" % envoy_config)
+                    logger.info("Configuration %s valid" % envoy_config)
                     return envoy_config
             except subprocess.CalledProcessError:
-                print ("Invalid envoy config")
+                logger.info("Invalid envoy config")
                 with open(envoy_config) as fd:
-                    print(fd.read())
+                    logger.info(fd.read())
         else:
-            print("Could not generate new Envoy configuration: %s" % rc.error)
-            print("Raw template output:")
-            print("%s" % rc.raw)
+            logger.info("Could not generate new Envoy configuration: %s" % rc.error)
+            logger.info("Raw template output:")
+            logger.info("%s" % rc.raw)
 
         raise ValueError("Unable to generate config")
 
@@ -147,7 +165,7 @@ def update_from_service(self, svc):
             self.update(key, config)
 
     def update(self, key, config):
-        print("update: including key %s" % key)
+        logger.info("update: including key %s" % key)
 
         with self.mutex:
             need_update = False
@@ -171,7 +189,7 @@ def delete(self, svc):
     def poke(self):
         with self.mutex:
             if self.processed == self.pokes:
-                print ("Scheduling restart")
+                logger.info("Scheduling restart")
             self.pokes += 1
 
 
@@ -194,7 +212,7 @@ def kube_v1():
             k8s_api = client.CoreV1Api()
         except FileNotFoundError:
             # Meh, just ride through.
-            print("No K8s")
+            logger.info("No K8s")
             pass
 
     return k8s_api
@@ -225,7 +243,7 @@ def read_cert_secret(k8s_api, secret_name, namespace):
         if e.reason == "Not Found":
             pass
         else:
-            print("secret %s/%s could not be read: %s" % (namespace, secret_name, e))
+            logger.info("secret %s/%s could not be read: %s" % (namespace, secret_name, e))
 
     if cert_data and cert_data.data:
         cert_data = cert_data.data
@@ -263,7 +281,7 @@ def sync(restarter):
 
             if config_data:
                 for key, config_yaml in config_data.data.items():
-                    # print("ambassador-config: found %s" % key)
+                    # logger.info("ambassador-config: found %s" % key)
                     restarter.update(key, config_yaml)
 
         # If we don't already see a TLS server key in its usual spot...
@@ -307,7 +325,7 @@ def sync(restarter):
         for svc in v1.list_namespaced_service(restarter.namespace).items:
             restarter.update_from_service(svc)
 
-    print ("Changes detected, regenerating envoy config.")
+    logger.info("Changes detected, regenerating envoy config.")
     restarter.restart()
 
 def watch_loop(restarter):
@@ -316,7 +334,7 @@ def watch_loop(restarter):
     if v1:
         w = watch.Watch()
         for evt in w.stream(v1.list_service_for_all_namespaces):
-            print("Event: %s %s/%s" % (evt["type"], 
+            logger.info("Event: %s %s/%s" % (evt["type"], 
                                        evt["object"].metadata.namespace, evt["object"].metadata.name))
             sys.stdout.flush()
 
@@ -325,7 +343,7 @@ def watch_loop(restarter):
             else:
                 restarter.update_from_service(evt["object"])
     else:
-        print("No K8s, idling")
+        logger.info("No K8s, idling")
         while True:
             time.sleep(60)
 
@@ -408,7 +426,6 @@ def main(mode, ambassador_config_dir, envoy_config_file, delay, pid):
     elif mode == "watch":
         restarter.start()
 
-
         while True:
             try:
                 # this is in a loop because sometimes the auth expires
@@ -417,7 +434,7 @@ def main(mode, ambassador_config_dir, envoy_config_file, delay, pid):
             except KeyboardInterrupt:
                 raise
             except:
-                traceback.print_exc()
+                logging.exception("could not watch for Kubernetes service changes")
     else:
          raise ValueError(mode)
 


Test patch: