Repository: scoutapp/scout_apm_python
Pull request #71 (2018-07-03 20:14:29)

InstrumentManager
Still needs to figure out already-installed, dont-install, and figure out who calls install on the instruments.

Also maybe use this to install django / flask / etc. instruments if possible?


Patch:
diff --git a/.travis.yml b/.travis.yml
index 5ccefe2..608668d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,8 +4,6 @@ matrix:
     - python: 3.4
     - python: 3.5
     - python: 3.6
-    - python: 3.7
-      dist: xenial
     - python: "pypy3"
 
 cache: pip
diff --git a/src/scout_apm/bottle/__init__.py b/src/scout_apm/bottle/__init__.py
index c0023da..c9fa3c9 100644
--- a/src/scout_apm/bottle/__init__.py
+++ b/src/scout_apm/bottle/__init__.py
@@ -30,7 +30,7 @@ class ScoutPlugin(object):
         scout_apm.core.install()
 
     def apply(self, callback, context):
-        if AgentContext.instance().config.value('monitor') is not True:
+        if AgentContext.instance.config.value('monitor') is not True:
             return callback
 
         def wrapper(*args, **kwargs):
diff --git a/src/scout_apm/core/__init__.py b/src/scout_apm/core/__init__.py
index 747028a..f932629 100644
--- a/src/scout_apm/core/__init__.py
+++ b/src/scout_apm/core/__init__.py
@@ -4,20 +4,28 @@ from __future__ import absolute_import
 import logging
 from os import getpid
 
+from scout_apm.core.config.config import ScoutConfig
 from scout_apm.core.context import AgentContext
 from scout_apm.core.core_agent_manager import CoreAgentManager
+from scout_apm.core.instrument_manager import InstrumentManager
 from scout_apm.core.metadata import AppMetadata
 
 logger = logging.getLogger(__name__)
 
 
 def install():
-    if not AgentContext.instance().config.value('monitor'):
+    config = ScoutConfig()
+    context = AgentContext.build(config=config)
+
+    if not context.config.value('monitor'):
         logger.info("APM Not Launching on PID: %s - Configuration 'monitor' is not true", getpid())
         return False
 
+    InstrumentManager().install_all()
+
     logger.debug('APM Launching on PID: %s', getpid())
     CoreAgentManager().launch()
+
     AppMetadata.report()
     AgentContext.socket().stop()
     return True
diff --git a/src/scout_apm/core/config/config.py b/src/scout_apm/core/config/config.py
index bf67943..9477af0 100644
--- a/src/scout_apm/core/config/config.py
+++ b/src/scout_apm/core/config/config.py
@@ -55,6 +55,7 @@ class ScoutConfig():
             'core_agent_download',
             'core_agent_launch',
             'core_agent_version',
+            'disabled_instruments',
             'download_url',
             'framework',
             'framework_version',
@@ -175,6 +176,7 @@ class ScoutConfigDefaults():
                 'log_level': 'info',
                 'name': '',
                 'monitor': False,
+                'disabled_instruments': [],
                 'socket_path': '{}/scout_apm_core-{}-{}-{}/core-agent.sock'.format(self.core_agent_dir,
                                                                                    self.core_agent_version,
                                                                                    ScoutConfig.arch(),
@@ -217,8 +219,22 @@ class BooleanConversion():
         return False
 
 
+class ListConversion():
+    @classmethod
+    def convert(cls, value):
+        if isinstance(value, list):
+            return value
+        if isinstance(value, tuple):
+            return list(value)
+        if isinstance(value, str):
+            # Split on commas
+            return value.split(',')
+        # Unknown type - default to empty?
+        return []
+
 CONVERSIONS = {
     'core_agent_download': BooleanConversion,
     'core_agent_launch': BooleanConversion,
     'monitor': BooleanConversion,
+    'disabled_instruments': ListConversion,
 }
diff --git a/src/scout_apm/core/context.py b/src/scout_apm/core/context.py
index 54f6a80..48336a7 100644
--- a/src/scout_apm/core/context.py
+++ b/src/scout_apm/core/context.py
@@ -3,16 +3,23 @@ import logging
 
 from .config.config import ScoutConfig
 from .socket import CoreAgentSocket
-from .thread_local import ThreadLocalSingleton
 
 # Logging
 logger = logging.getLogger(__name__)
 
 
-class AgentContext(ThreadLocalSingleton):
+class AgentContext():
+    instance = None
+
     def __init__(self, *args, **kwargs):
         self.config = kwargs.get('config', ScoutConfig())
 
+    @classmethod
+    def build(cls, *args, **kwargs):
+        cls.instance = AgentContext(*args, **kwargs)
+        return cls.instance
+
     @classmethod
     def socket(cls):
         return CoreAgentSocket.instance(scout_config=ScoutConfig())
+
diff --git a/src/scout_apm/core/core_agent_manager.py b/src/scout_apm/core/core_agent_manager.py
index 39c1fbe..58cae80 100644
--- a/src/scout_apm/core/core_agent_manager.py
+++ b/src/scout_apm/core/core_agent_manager.py
@@ -23,19 +23,19 @@ class CoreAgentManager:
     def __init__(self):
         self.core_agent_bin_path = None
         self.core_agent_bin_version = None
-        self.core_agent_dir = '{}/{}'.format(AgentContext.instance().config.value('core_agent_dir'),
-                                             AgentContext.instance().config.core_agent_full_name())
+        self.core_agent_dir = '{}/{}'.format(AgentContext.instance.config.value('core_agent_dir'),
+                                             AgentContext.instance.config.core_agent_full_name())
         self.downloader = CoreAgentDownloader(self.core_agent_dir,
-                                              AgentContext.instance().config.core_agent_full_name())
+                                              AgentContext.instance.config.core_agent_full_name())
 
     def launch(self):
-        if AgentContext.instance().config.value('core_agent_launch') is not True:
+        if AgentContext.instance.config.value('core_agent_launch') is not True:
             logger.debug("Not attempting to launch Core Agent "
                          "due to 'core_agent_launch' setting.")
             return
 
         if self.verify() is not True:
-            if AgentContext.instance().config.value('core_agent_download') is True:
+            if AgentContext.instance.config.value('core_agent_download') is True:
                 self.download()
             else:
                 logger.debug("Not attempting to download Core Agent due "
@@ -75,22 +75,22 @@ class CoreAgentManager:
         return ['--daemonize', 'true']
 
     def socket_path(self):
-        socket_path = AgentContext.instance().config.value('socket_path')
+        socket_path = AgentContext.instance.config.value('socket_path')
         return ['--socket', socket_path]
 
     def log_level(self):
-        level = AgentContext.instance().config.value('log_level')
+        level = AgentContext.instance.config.value('log_level')
         return ['--log-level', level]
 
     def log_file(self):
-        path = AgentContext.instance().config.value('log_file')
+        path = AgentContext.instance.config.value('log_file')
         if path is not None:
             return ['--log-file', path]
         else:
             return []
 
     def config_file(self):
-        path = AgentContext.instance().config.value('config_file')
+        path = AgentContext.instance.config.value('config_file')
         if path is not None:
             return ['--config-file', path]
         else:
@@ -188,7 +188,7 @@ class CoreAgentDownloader():
                 core_agent_full_name=self.core_agent_full_name)
 
     def root_url(self):
-        return AgentContext.instance().config.value('download_url')
+        return AgentContext.instance.config.value('download_url')
 
 
 class CoreAgentManifest:
diff --git a/src/scout_apm/core/instrument_manager.py b/src/scout_apm/core/instrument_manager.py
new file mode 100644
index 0000000..385125c
--- /dev/null
+++ b/src/scout_apm/core/instrument_manager.py
@@ -0,0 +1,36 @@
+import importlib
+import logging
+
+from scout_apm.core.context import AgentContext
+
+
+logger = logging.getLogger(__name__)
+
+
+class InstrumentManager:
+    INSTRUMENT_NAMESPACE = 'scout_apm.instruments'
+    DEFAULT_INSTRUMENTS = ['jinja2', 'pymongo', 'redis', 'urllib3']
+
+    def install(self, module_name, klass="Instrument", *args, **kwargs):
+        try:
+            installable = importlib.import_module(module_name)
+            installable = getattr(installable, klass)
+            installable = installable(*args, **kwargs)
+
+            result = getattr(installable, 'install')()
+            return result
+        except Exception as e:
+            logger.info('Exception while installing instrument {}: {}'.format(module_name, repr(e)))
+            return False
+
+    def install_all(self):
+        for module_name in self.__class__.DEFAULT_INSTRUMENTS:
+            if self.is_disabled(module_name):
+                logger.info('{} instruments are disabled. Skipping.'.format(module_name))
+                continue
+            self.install('{}.{}'.format(self.__class__.INSTRUMENT_NAMESPACE, module_name))
+
+    def is_disabled(self, module_name):
+        disabled = AgentContext.instance.config.value('disabled_instruments')
+        if module_name in disabled:
+            return True
diff --git a/src/scout_apm/core/metadata.py b/src/scout_apm/core/metadata.py
index 5015d6a..2e86af6 100644
--- a/src/scout_apm/core/metadata.py
+++ b/src/scout_apm/core/metadata.py
@@ -34,18 +34,18 @@ class AppMetadata():
                                                            version_tuple[1],
                                                            version_tuple[2]),
                     'server_time':        datetime.utcnow().isoformat() + 'Z',
-                    'framework':          AgentContext.instance().config.value('framework'),
-                    'framework_version':  AgentContext.instance().config.value('framework_version'),
+                    'framework':          AgentContext.instance.config.value('framework'),
+                    'framework_version':  AgentContext.instance.config.value('framework_version'),
                     'environment':        '',
-                    'app_server':         AgentContext.instance().config.value('app_server'),
-                    'hostname':           AgentContext.instance().config.value('hostname'),
+                    'app_server':         AgentContext.instance.config.value('app_server'),
+                    'hostname':           AgentContext.instance.config.value('hostname'),
                     'database_engine':    '',  # Detected
                     'database_adapter':   '',  # Raw
                     'application_name':   '',  # Environment.application_name,
                     'libraries':          cls.get_python_packages_versions(),
                     'paas':               '',
-                    'application_root':   AgentContext.instance().config.value('application_root'),
-                    'git_sha':            AgentContext.instance().config.value('revision_sha')}
+                    'application_root':   AgentContext.instance.config.value('application_root'),
+                    'git_sha':            AgentContext.instance.config.value('revision_sha')}
         except Exception as e:
             logger.debug('Exception in AppMetadata: %s', repr(e))
 
diff --git a/src/scout_apm/instruments/__init__.py b/src/scout_apm/instruments/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/scout_apm/instruments/jinja2.py b/src/scout_apm/instruments/jinja2.py
new file mode 100644
index 0000000..eb8ec8b
--- /dev/null
+++ b/src/scout_apm/instruments/jinja2.py
@@ -0,0 +1,51 @@
+from __future__ import absolute_import
+
+import logging
+
+from scout_apm.core.tracked_request import TrackedRequest
+from scout_apm.core.monkey import monkeypatch_method
+
+logger = logging.getLogger(__name__)
+
+
+class Instrument:
+    def __init__(self):
+        self.installed = False
+
+    def installable(self):
+        try:
+            from jinja2 import Template
+        except ImportError:
+            logger.info("Unable to import for Jinja2 instruments")
+            return False
+        if self.installed:
+            logger.warn("Jinja2 Instruments are already installed.")
+            return False
+        return True
+
+    def install(self):
+        if not self.installable():
+            logger.info("Jinja2 instruments are not installable. Skipping.")
+            return False
+
+        self.installed = True
+
+        try:
+            from jinja2 import Template
+
+            @monkeypatch_method(Template)
+            def render(original, self, *args, **kwargs):
+                tr = TrackedRequest.instance()
+                span = tr.start_span(operation='Template/Render')
+                span.tag('name', self.name)
+
+                try:
+                    return original(*args, **kwargs)
+                finally:
+                    tr.stop_span()
+
+            logger.info("Instrumented Jinja2")
+
+        except Exception as e:
+            logger.warn('Unable to instrument for Jinja2 Template.render: {}'.format(repr(e)))
+        return True
diff --git a/src/scout_apm/instruments/pymongo.py b/src/scout_apm/instruments/pymongo.py
new file mode 100644
index 0000000..180634c
--- /dev/null
+++ b/src/scout_apm/instruments/pymongo.py
@@ -0,0 +1,72 @@
+from __future__ import absolute_import
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+from scout_apm.core.tracked_request import TrackedRequest
+from scout_apm.core.monkey import monkeypatch_method
+
+class Instrument:
+    PYMONGO_METHODS = ['aggregate', 'bulk_write', 'count',
+                       'create_index', 'create_indexes', 'delete_many',
+                       'delete_one', 'distinct', 'drop',
+                       'drop_index', 'drop_indexes', 'ensure_index',
+                       'find_and_modify', 'find_one', 'find_one_and_delete',
+                       'find_one_and_replace', 'find_one_and_update', 'group',
+                       'inline_map_reduce', 'insert', 'insert_many',
+                       'insert_one', 'map_reduce', 'reindex',
+                       'remove', 'rename', 'replace_one',
+                       'save', 'update', 'update_many',
+                       'update_one']
+
+    def __init__(self):
+        self.installed = False
+
+    def installable(self):
+        try:
+            from pymongo.collection import Collection
+        except ImportError:
+            logger.info("Unable to import for PyMongo instruments")
+            return False
+        if self.installed:
+            logger.warn("PyMongo Instruments are already installed.")
+            return False
+        return True
+
+
+    def install(self):
+        if not self.installable():
+            logger.info("PyMongo instruments are not installable. Skipping.")
+            return False
+
+        self.installed = True
+
+        try:
+            from pymongo.collection import Collection
+        except ImportError:
+            logger.info("Unable to import for PyMongo instruments. Instrument install failed.")
+            return False
+
+        for method_str in self.__class__.PYMONGO_METHODS:
+            try:
+                code_str = """
+@monkeypatch_method(Collection)
+def {method_str}(original, self, *args, **kwargs):
+    tr = TrackedRequest.instance()
+    span = tr.start_span(operation='MongoDB/{camel_name}')
+    span.tag('name', self.name)
+
+    try:
+        return original(*args, **kwargs)
+    finally:
+        tr.stop_span()
+""".format(method_str=method_str, camel_name=''.join(c.title() for c in method_str.split('_')))
+
+                exec(code_str)
+                logger.info('Instrumented PyMongo Collection.{}'.format(method_str))
+
+            except Exception as e:
+                logger.warn('Unable to instrument for PyMongo Collection.{}: {}'.format(method_str, repr(e)))
+        return True
diff --git a/src/scout_apm/instruments/redis.py b/src/scout_apm/instruments/redis.py
new file mode 100644
index 0000000..1fccaa4
--- /dev/null
+++ b/src/scout_apm/instruments/redis.py
@@ -0,0 +1,75 @@
+from __future__ import absolute_import
+
+import logging
+
+from scout_apm.core.tracked_request import TrackedRequest
+from scout_apm.core.monkey import monkeypatch_method
+
+logger = logging.getLogger(__name__)
+
+
+class Instrument:
+    def __init__(self):
+        self.installed = False
+
+    def installable(self):
+        try:
+            from redis import StrictRedis
+            from redis.client import BasePipeline
+        except ImportError:
+            logger.info("Unable to import for Redis instruments")
+            return False
+        if self.installed:
+            logger.warn("Redis Instruments are already installed.")
+            return False
+        return True
+
+    def install(self):
+        if not self.installable():
+            logger.info("Redis instruments are not installable. Skipping.")
+            return False
+
+        self.installed = True
+
+        self.patch_strictredis()
+        self.patch_basepipeline()
+
+        logger.info("Instrumented Redis")
+        return True
+
+    def patch_strictredis(self):
+        try:
+            from redis import StrictRedis
+
+            @monkeypatch_method(StrictRedis)
+            def execute_command(original, self, *args, **kwargs):
+                try:
+                    op = args[0]
+                except(IndexError, TypeError):
+                    op = 'Unknown'
+
+                tr = TrackedRequest.instance()
+                tr.start_span(operation='Redis/{}'.format(op))
+
+                try:
+                    return original(*args, **kwargs)
+                finally:
+                    tr.stop_span()
+        except Exception as e:
+            logger.warn('Unable to instrument for Redis StrictRedis.execute_command: {}'.format(repr(e)))
+
+    def patch_basepipeline(self):
+        try:
+            from redis.client import BasePipeline
+
+            @monkeypatch_method(BasePipeline)
+            def execute(original, self, *args, **kwargs):
+                tr = TrackedRequest.instance()
+                tr.start_span(operation='Redis/MULTI')
+
+                try:
+                    return original(*args, **kwargs)
+                finally:
+                    tr.stop_span()
+        except Exception as e:
+            logger.warn('Unable to instrument for Redis BasePipeline.execute: {}'.format(repr(e)))
diff --git a/src/scout_apm/instruments/urllib3.py b/src/scout_apm/instruments/urllib3.py
new file mode 100644
index 0000000..86bc544
--- /dev/null
+++ b/src/scout_apm/instruments/urllib3.py
@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+
+import logging
+
+from scout_apm.core.tracked_request import TrackedRequest
+from scout_apm.core.monkey import monkeypatch_method
+
+logger = logging.getLogger(__name__)
+
+
+class Instrument:
+    def __init__(self):
+        self.installed = False
+
+    def installable(self):
+        try:
+            from urllib3 import HTTPConnectionPool, PoolManager
+        except ImportError:
+            logger.info("Unable to import for Urllib3 instruments")
+            return False
+        if self.installed:
+            logger.warn("Urllib3 Instruments are already installed.")
+            return False
+        return True
+
+    def install(self):
+        if not self.installable():
+            logger.info("Urllib3 instruments are not installable. Skipping.")
+            return False
+
+        self.installed = True
+
+        self.patch_connectionpool()
+
+        logger.info("Instrumented Urllib3")
+        return True
+
+    def patch_connectionpool(self):
+        try:
+            from urllib3 import HTTPConnectionPool
+
+            @monkeypatch_method(HTTPConnectionPool)
+            def urlopen(original, self, *args, **kwargs):
+                method = 'Unknown'
+                url = 'Unknown'
+                try:
+                    if 'method' in kwargs:
+                        method = kwargs['method']
+                    else:
+                        method = args[0]
+                    url = '{}'.format(self._absolute_url('/'))
+                except Exception as e:
+                    logger.error('Could not get instrument data for HTTPConnectionPool: {}'.format(repr(e)))
+
+                tr = TrackedRequest.instance()
+                span = tr.start_span(operation='HTTP/{}'.format(method))
+                span.tag('url', '{}'.format(url))
+
+                try:
+                    return original(*args, **kwargs)
+                finally:
+                    tr.stop_span()
+        except Exception as e:
+            logger.warn('Unable to instrument for Urllib3 HTTPConnectionPool.urlopen: {}'.format(repr(e)))


Test patch:
diff --git a/tests/scout_apm_tests/instrument_manager_test.py b/tests/scout_apm_tests/instrument_manager_test.py
new file mode 100644
index 0000000..92b0265
--- /dev/null
+++ b/tests/scout_apm_tests/instrument_manager_test.py
@@ -0,0 +1,74 @@
+from scout_apm.core.config.config import ScoutConfig
+from scout_apm.core.instrument_manager import InstrumentManager
+
+
+# Fake instrument - note, this uses a module level global var to track if it's
+# been installed. Reset it after your test to reuse.
+klassy_instrument_pos_arg = False
+klassy_instrument_kwarg = False
+class KlassyInstrument():
+    def __init__(self, pos_arg, kwarg=None):
+        self.pos_arg = pos_arg
+        self.kwarg = kwarg
+
+    def install(self):
+        global klassy_instrument_pos_arg
+        global klassy_instrument_kwarg
+
+        klassy_instrument_pos_arg = self.pos_arg
+        klassy_instrument_kwarg = self.kwarg
+        return True
+
+# Fake instrument - note, this uses a module level global var to track if it's
+# been installed. Reset it after your test to reuse.
+class ExceptionalInstrument():
+    def __init__(self):
+        pass
+
+    def install(self):
+        raise Exception("foo")
+
+
+# Default instrument name
+default_instrument_installed = False
+class Instrument():
+    def install(self):
+        global default_instrument_installed
+
+        default_instrument_installed = True
+        return True
+
+
+def test_loads_default_instrument():
+    global default_instrument_installed
+    assert(not default_instrument_installed)
+
+    result = InstrumentManager().install('scout_apm_tests.instrument_manager_test')
+
+    assert(default_instrument_installed)
+    assert(result)
+
+    default_instrument_installed = False
+
+
+def test_loads_class_instrument():
+    global klassy_instrument_pos_arg
+    global klassy_instrument_kwarg
+
+    assert(not klassy_instrument_pos_arg)
+    assert(not klassy_instrument_kwarg)
+
+    result = InstrumentManager().install('scout_apm_tests.instrument_manager_test', 'KlassyInstrument', 'pos arg', kwarg='kwarg')
+
+    assert(klassy_instrument_pos_arg == 'pos arg')
+    assert(klassy_instrument_kwarg == 'kwarg')
+    assert(result)
+
+    klassy_instrument_pos_arg = False
+    klassy_instrument_kwarg = False
+
+
+def test_handles_exception():
+    result = InstrumentManager().install('scout_apm_tests.instrument_manager_test', 'ExceptionalInstrument')
+    assert(not result)
+