Repository: DataDog/dd-trace-py

Issue:
boto3 patch method crashes while fetching AWS region
The `boto3` patch method of the `ddtrace` library expects the boto3 request to have the region parameter set.

Some of requests do not have it set (shared an example with Datadog engineers privately) so the patched function crashes on the following line:

https://github.com/DataDog/dd-trace-py/blob/7a3d0f69610a160bfd71cec8caa8366bc6977a8e/ddtrace/contrib/boto/patch.py#L68


Discussion:

Resolving patch:
diff --git a/ddtrace/contrib/boto/patch.py b/ddtrace/contrib/boto/patch.py
--- a/ddtrace/contrib/boto/patch.py
+++ b/ddtrace/contrib/boto/patch.py
@@ -11,10 +11,18 @@
 _Boto_client = boto.connection.AWSQueryConnection
 
 SPAN_TYPE = "boto"
-AWS_QUERY_ARGS_NAME = ('operation_name', 'params', 'path', 'verb')
-AWS_AUTH_ARGS_NAME = ('method', 'path', 'headers', 'data', 'host', 'auth_path', 'sender')
-AWS_QUERY_TRACED_ARGS = ['operation_name', 'params', 'path']
-AWS_AUTH_TRACED_ARGS = ['path', 'data', 'host']
+AWS_QUERY_ARGS_NAME = ("operation_name", "params", "path", "verb")
+AWS_AUTH_ARGS_NAME = (
+    "method",
+    "path",
+    "headers",
+    "data",
+    "host",
+    "auth_path",
+    "sender",
+)
+AWS_QUERY_TRACED_ARGS = ["operation_name", "params", "path"]
+AWS_AUTH_TRACED_ARGS = ["path", "data", "host"]
 
 
 def patch():
@@ -23,21 +31,29 @@ def patch():
     different services for connection. For exemple EC2 uses AWSQueryConnection and
     S3 uses AWSAuthConnection
     """
-    if getattr(boto.connection, '_datadog_patch', False):
+    if getattr(boto.connection, "_datadog_patch", False):
         return
-    setattr(boto.connection, '_datadog_patch', True)
-
-    wrapt.wrap_function_wrapper('boto.connection', 'AWSQueryConnection.make_request', patched_query_request)
-    wrapt.wrap_function_wrapper('boto.connection', 'AWSAuthConnection.make_request', patched_auth_request)
-    Pin(service="aws", app="aws", app_type="web").onto(boto.connection.AWSQueryConnection)
-    Pin(service="aws", app="aws", app_type="web").onto(boto.connection.AWSAuthConnection)
+    setattr(boto.connection, "_datadog_patch", True)
+
+    wrapt.wrap_function_wrapper(
+        "boto.connection", "AWSQueryConnection.make_request", patched_query_request
+    )
+    wrapt.wrap_function_wrapper(
+        "boto.connection", "AWSAuthConnection.make_request", patched_auth_request
+    )
+    Pin(service="aws", app="aws", app_type="web").onto(
+        boto.connection.AWSQueryConnection
+    )
+    Pin(service="aws", app="aws", app_type="web").onto(
+        boto.connection.AWSAuthConnection
+    )
 
 
 def unpatch():
-    if getattr(boto.connection, '_datadog_patch', False):
-        setattr(boto.connection, '_datadog_patch', False)
-        unwrap(boto.connection.AWSQueryConnection, 'make_request')
-        unwrap(boto.connection.AWSAuthConnection, 'make_request')
+    if getattr(boto.connection, "_datadog_patch", False):
+        setattr(boto.connection, "_datadog_patch", False)
+        unwrap(boto.connection.AWSQueryConnection, "make_request")
+        unwrap(boto.connection.AWSAuthConnection, "make_request")
 
 
 # ec2, sqs, kinesis
@@ -47,32 +63,38 @@ def patched_query_request(original_func, instance, args, kwargs):
     if not pin or not pin.enabled():
         return original_func(*args, **kwargs)
 
-    endpoint_name = getattr(instance, "host").split('.')[0]
+    endpoint_name = getattr(instance, "host").split(".")[0]
 
-    with pin.tracer.trace('{}.command'.format(endpoint_name), service="{}.{}".format(pin.service, endpoint_name),
-                          span_type=SPAN_TYPE) as span:
+    with pin.tracer.trace(
+        "{}.command".format(endpoint_name),
+        service="{}.{}".format(pin.service, endpoint_name),
+        span_type=SPAN_TYPE,
+    ) as span:
 
         operation_name = None
         if args:
             operation_name = args[0]
-            span.resource = '%s.%s' % (endpoint_name, operation_name.lower())
+            span.resource = "%s.%s" % (endpoint_name, operation_name.lower())
         else:
             span.resource = endpoint_name
 
         # Adding the args in AWS_QUERY_TRACED_ARGS if exist to the span
         if not aws.is_blacklist(endpoint_name):
-            for arg in aws.unpacking_args(args, AWS_QUERY_ARGS_NAME, AWS_QUERY_TRACED_ARGS):
+            for arg in aws.unpacking_args(
+                args, AWS_QUERY_ARGS_NAME, AWS_QUERY_TRACED_ARGS
+            ):
                 span.set_tag(arg[0], arg[1])
 
         # Obtaining region name
-        region = getattr(instance, "region")
-        region_name = get_region_name(region)
+        region_name = _get_instance_region_name(instance)
 
         meta = {
-            'aws.agent': 'boto',
-            'aws.operation': operation_name,
-            'aws.region': region_name,
+            aws.AGENT: "boto",
+            aws.OPERATION: operation_name,
         }
+        if region_name:
+            meta[aws.REGION] = region_name
+
         span.set_tags(meta)
 
         # Original func returns a boto.connection.HTTPResponse object
@@ -97,31 +119,37 @@ def patched_auth_request(original_func, instance, args, kwargs):
     if not pin or not pin.enabled():
         return original_func(*args, **kwargs)
 
-    endpoint_name = getattr(instance, "host").split('.')[0]
+    endpoint_name = getattr(instance, "host").split(".")[0]
 
-    with pin.tracer.trace('{}.command'.format(endpoint_name), service="{}.{}".format(pin.service, endpoint_name),
-                          span_type=SPAN_TYPE) as span:
+    with pin.tracer.trace(
+        "{}.command".format(endpoint_name),
+        service="{}.{}".format(pin.service, endpoint_name),
+        span_type=SPAN_TYPE,
+    ) as span:
 
         # Adding the args in AWS_AUTH_TRACED_ARGS if exist to the span
         if not aws.is_blacklist(endpoint_name):
-            for arg in aws.unpacking_args(args, AWS_AUTH_ARGS_NAME, AWS_AUTH_TRACED_ARGS):
+            for arg in aws.unpacking_args(
+                args, AWS_AUTH_ARGS_NAME, AWS_AUTH_TRACED_ARGS
+            ):
                 span.set_tag(arg[0], arg[1])
 
         if args:
             http_method = args[0]
-            span.resource = '%s.%s' % (endpoint_name, http_method.lower())
+            span.resource = "%s.%s" % (endpoint_name, http_method.lower())
         else:
             span.resource = endpoint_name
 
         # Obtaining region name
-        region = getattr(instance, "region", None)
-        region_name = get_region_name(region)
+        region_name = _get_instance_region_name(instance)
 
         meta = {
-            'aws.agent': 'boto',
-            'aws.operation': operation_name,
-            'aws.region': region_name,
+            aws.AGENT: "boto",
+            aws.OPERATION: operation_name,
         }
+        if region_name:
+            meta[aws.REGION] = region_name
+
         span.set_tags(meta)
 
         # Original func returns a boto.connection.HTTPResponse object
@@ -132,7 +160,9 @@ def patched_auth_request(original_func, instance, args, kwargs):
         return result
 
 
-def get_region_name(region):
+def _get_instance_region_name(instance):
+    region = getattr(instance, "region", None)
+
     if not region:
         return None
     if isinstance(region, str):
diff --git a/ddtrace/ext/aws.py b/ddtrace/ext/aws.py
--- a/ddtrace/ext/aws.py
+++ b/ddtrace/ext/aws.py
@@ -25,3 +25,8 @@ def unpacking_args(args, args_name, traced_args_list):
             response += [(args_name[index], arg)]
         index += 1
     return response
+
+
+REGION = "aws.region"
+AGENT = "aws.agent"
+OPERATION = "aws.operation"


Test patch: