Repository: DataDog/dd-trace-py

Issue:
Unexpected warning of cannot initialise DDWAF object with a UUID
<!--
Thanks for taking the time for reporting an issue!

Before reporting an issue on dd-trace-py, please be sure to provide all
necessary information.

If you're hitting a bug, make sure that you're using the latest version of this
library.
-->

### Summary of problem

With Datadog ASM enabled for a Flask app, calling a URL with a valid UUID in the path results a warning to be logged by the ddtrace library. The warning is related to WAF. I expect no warning and for ddtrace to handle UUIDs in a URL path.

### Which version of dd-trace-py are you using?

1.10.1

### Which version of pip are you using?

23.0

### Which libraries and their versions are you using?

<details>
  <summary>`pip freeze`</summary>

aiohttp==3.8.4
aiosignal==1.3.1
alembic==1.4.3
algoliasearch==2.5.0
altair==4.2.2
analytics-python==1.4.0
aniso8601==9.0.1
apispec==4.7.1
appnope==0.1.0
arrow==1.2.3
asn1crypto==1.5.1
asttokens==2.2.1
async-timeout==4.0.2
attrs==22.2.0
Authlib==1.0.1
backcall==0.2.0
backoff==1.10.0
bandit==1.7.0
basicauth==0.4.1
bcrypt==3.2.0
beautifulsoup4==4.10.0
binaryornot==0.4.4
black==22.8.0
bleach==5.0.1
blinker==1.4
Bootstrap-Flask==2.1.0
boto3==1.20.21
botocore==1.23.21
bs4==0.0.1
bytecode==0.14.0
CacheControl==0.12.11
cached-property==1.5.2
cachelib==0.10.2
cachetools==4.2.4
cairocffi==1.5.0
CairoSVG==2.7.0
cattrs==22.2.0
certifi==2022.12.7
cffi==1.15.1
cfgv==3.3.1
chardet==4.0.0
charset-normalizer==2.0.12
click==8.1.3
cloudflare==2.11.1
codeowners==0.4.1
colorama==0.4.4
commonmark==0.9.1
contentful==1.13.1
contentful-management==2.11.0
contextlib2==21.6.0
cookiecutter==2.1.1
coverage==6.5.0
cron-descriptor==1.2.31
cryptography==36.0.2
cssbeautifier==1.14.7
cssselect==1.2.0
cssselect2==0.7.0
customerio==1.5
datadog==0.42.0
datasets==2.10.1
dbfread==2.0.7
ddsketch==2.0.4
ddtrace==1.10.1
decorator==5.1.1
deepmerge==0.3.0
defusedcsv==2.0.0
defusedxml==0.7.1
Deprecated==1.2.13
deprecation==2.1.0
deptry==0.7.0
diceware==0.9.6
dictdiffer==0.9.0
dill==0.3.6
distlib==0.3.6
djlint==1.18.0
ecdsa==0.18.0
editdistance==0.5.3
EditorConfig==0.12.3
elasticsearch==7.13.4
elementpath==3.0.2
entrypoints==0.4
envier==0.4.0
environs==9.3.5
eradicate==2.2.0
et-xmlfile==1.1.0
exceptiongroup==1.1.1
executing==1.2.0
factory-boy==3.2.0
Faker==13.15.0
fakeredis==2.7.1
fastdiff==0.3.0
filelock==3.10.0
filetype==1.2.0
firebase-admin==4.5.3
FixedWidth==1.3
flake8==5.0.4
flake8-assertive==2.1.0
flake8-bugbear==22.8.23
flake8-cached==0.2.0
flake8-comprehensions==3.8.0
flake8-eradicate==1.3.0
flake8-logging-format==0.7.5
flake8-string-format==0.3.0
flake8-tidy-imports==4.6.0
flameprof==0.4
Flask==1.1.2
Flask-Admin==1.6.0
Flask-Bcrypt==1.0.1
Flask-Caching==2.0.1
Flask-Cors==3.0.10
Flask-Dance==5.1.0
Flask-DebugToolbar==0.13.1
Flask-HTTPAuth==3.3.0
Flask-Login==0.6.2
Flask-RESTful==0.3.9
flask-saml2==0.3.0
flask-shell-ipython==0.4.1
flask-smorest==0.30.0
Flask-SQLAlchemy==2.5.1
Flask-Threads==0.1.0
Flask-WTF==1.0.1
flatdict==4.0.1
freezegun==1.1.0
frozenlist==1.3.3
fsspec==2023.3.0
gitdb==4.0.10
GitPython==3.1.31
google-api-core==1.30.0
google-api-python-client==2.39.0
google-auth==1.32.1
google-auth-httplib2==0.1.0
google-auth-oauthlib==0.4.6
google-cloud-core==2.2.2
google-cloud-firestore==2.3.4
google-cloud-speech==2.13.0
google-cloud-storage==2.1.0
google-cloud-vision==2.3.2
google-crc32c==1.5.0
google-pasta==0.2.0
google-resumable-media==2.4.1
googleapis-common-protos==1.58.0
googlemaps==4.4.5
gql==3.0.0b1
graphql-core==3.1.7
grpcio==1.47.0
grpcio-tools==1.47.0
gspread==5.6.2
gunicorn==20.1.0
guppy3==3.1.2
h2==2.6.2
hellosign-python-sdk==4.0.0
honcho==1.1.0
hpack==3.0.0
html-tag-names==0.1.2
html-void-elements==0.1.0
html5lib==1.1
httplib2==0.21.0
huggingface-hub==0.13.3
humanize==4.6.0
hunter==3.4.3
hyper==0.7.0
hyperframe==3.2.0
icalendar==4.1.0
identify==2.5.21
idna==2.10
import-deps==0.2.0
importlib-metadata==4.13.0
inflection==0.5.1
iniconfig==2.0.0
ipdb==0.13.11
ipython==8.10.0
iso8601==0.1.16
isodate==0.6.1
isort==5.10.1
itsdangerous==2.0.1
jedi==0.18.2
Jinja2==3.0.3
jinja2-time==0.2.0
jmespath==0.10.0
joblib==1.2.0
jsbeautifier==1.14.7
jsonlines==3.1.0
jsonpickle==2.2.0
jsonschema==4.17.3
jwcrypto==1.4
laboratory==1.0.2
langdetect==1.0.9
lazyasd==0.1.4
lml==0.1.0
lxml==4.9.1

marshmallow-dataclass==8.4.1
marshmallow-enum==1.5.1
marshmallow-jsonschema==0.11.1
matplotlib-inline==0.1.6
mccabe==0.7.0
mdurl==0.1.2
memory-profiler==0.61.0
monotonic==1.6
more-itertools==8.8.0
moto==4.1.5
msgpack==1.0.5
multidict==6.0.4
multiprocess==0.70.14
mypy==0.982
mypy-extensions==1.0.0
ndg-httpsclient==0.5.1
nodeenv==1.7.0
notion==0.0.28
numpy==1.22.0
oauthlib==3.2.2
openai==0.26.5
openpyxl==3.0.9
opentok==3.3.0
oscrypto==1.3.0
packaging==23.0
pandas==1.3.5
paramiko==3.1.0
parsel==1.7.0
parso==0.8.3
pathos==0.3.0
pathspec==0.10.3
pbr==5.11.1
pdf2image==1.16.0
pdfminer.six==20201018
pexpect==4.8.0
pickleshare==0.7.5
pikepdf==5.6.1
Pillow==9.3.0
pillow-heif==0.7.2
platformdirs==3.1.1
plotly==4.14.3
pluggy==1.0.0
PostgreSQL-Audit==0.11.1
pox==0.3.2
ppft==1.7.6.6
pre-commit==2.10.1
prompt-toolkit==3.0.38
proto-plus==1.22.2
protobuf==3.20.3
protobuf3-to-dict==0.1.5
psutil==5.9.4
psycopg2-binary==2.9.5
ptyprocess==0.7.0
pure-eval==0.2.2
pusher==3.3.2
pyarrow==11.0.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.9.1
pycparser==2.21
pycryptodome==3.17
pycryptodomex==3.17
pydeck==0.8.0
pyexcel==0.7.0
pyexcel-io==0.6.6
pyexcel-xls==0.7.0
pyexcel-xlsx==0.6.0
pyflakes==2.5.0
PyGithub==1.55
Pygments==2.14.0
PyJWT==2.4.0
Pympler==1.0.1
PyNaCl==1.5.0
pyOpenSSL==17.5.0
pyparsing==3.0.9
PyPDF2==2.11.0
pyphen==0.14.0
pyrsistent==0.19.3
pysftp==0.2.9
pytest==7.2.0
pytest-cov==4.0.0
pytest-mock==3.10.0
pytest-rerunfailures==10.2
pytest-subtests==0.8.0
pytest-testmon==1.3.7
pytest-unordered==0.5.1
python-dateutil==2.8.2
python-docx==0.8.11
python-dotenv==0.19.2
python-editor==1.0.4
python-http-client==3.3.1
python-intercom==3.1.0
python-jose==3.3.0
python-json-logger==2.0.7
python-keycloak==2.11.1
python-Levenshtein==0.12.2
python-redis-rate-limit==0.0.8
python-slugify==5.0.2
pytz==2021.1
pytz-deprecation-shim==0.1.0.post0
PyYAML==6.0
rapidfuzz==0.14.2
redis==4.4.2
Redis-Sentinel-Url==1.0.1
regex==2022.10.31
requests==2.25.1
requests-file==1.5.1
requests-oauthlib==1.3.1
requests-toolbelt==0.9.1
responses==0.13.4
retrying==1.3.4
rich==13.3.2
rq==1.11.0
rsa==4.9
s3fs==0.4.2
s3transfer==0.5.2
sagemaker==2.117.0
schema==0.7.5
scikit-learn==1.0.2
scipy==1.8.0
seeuletter==1.0.0
semantic-version==2.10.0
semver==2.13.0
sendgrid==6.8.1
sentry-sdk==1.9.9
setproctitle==1.3.1
signxml==3.1.0
simple-salesforce==1.11.4
six==1.16.0
slack-sdk==3.15.2
smdebug-rulesconfig==1.0.1
smmap==5.0.0
snapshottest==0.6.0
snowflake-connector-python==2.7.7
snowflake-sqlalchemy==1.2.4
sortedcontainers==2.4.0
soupsieve==2.4
spyne==2.13.16
SQLAlchemy==1.4.41
SQLAlchemy-Utils==0.38.3
sqlalchemy2-stubs==0.0.2a27
sqlparse==0.4.2
sqltap==0.3.11
stack-data==0.6.2
starkbank-ecdsa==2.2.0
stevedore==5.0.0
streamlit==1.12.2
stripe==4.2.0
structlog==22.3.0
structlog-sentry==2.0.0
tabulate==0.8.9
tenacity==8.2.2
termcolor==1.1.0
text-unidecode==1.3
texttable==1.6.7
threadpoolctl==3.1.0
tinycss2==1.2.1
tldextract==2.2.3
toml==0.10.2
tomli==2.0.1
toolz==0.12.0
tornado==6.2
tqdm==4.62.3
traitlets==5.9.0
treepoem==3.5.0
typeguard==2.13.3
types-backports==0.1.3
types-bleach==5.0.3.1
types-boto==2.49.18.6
types-cachetools==5.3.0.4
types-certifi==2021.10.8.3
types-chardet==4.0.4
types-click==7.1.8
types-cryptography==3.3.23.2
types-Flask==1.1.6
types-freezegun==1.1.9
types-google-cloud-ndb==1.12.0.0
types-Jinja2==2.11.9
types-Markdown==3.3.12
types-MarkupSafe==1.1.10
types-paramiko==2.10.0
types-protobuf==3.19.20
types-pyOpenSSL==22.1.0.2
types-python-dateutil==2.8.15
types-python-slugify==5.0.4
types-pytz==2022.7.1.2
types-PyYAML==6.0.4
types-redis==3.5.7
types-requests==2.27.25
types-setuptools==57.4.18
types-stripe==3.5.2.1
types-tabulate==0.8.9
types-termcolor==1.1.4
types-toml==0.10.8.5
types-urllib3==1.26.25.8
types-Werkzeug==1.0.9
typing-inspect==0.8.0
typing_extensions==4.3.0
tzdata==2022.7
tzlocal==4.3
uncalled==0.1.6
Unidecode==1.3.6
unittest-xml-reporting==3.0.4
uritemplate==4.1.1
urllib3==1.26.15
URLObject==2.4.3
validators==0.20.0
virtualenv==20.21.0
w3lib==2.1.1
wasmer==1.0.0
wasmer-compiler-cranelift==1.0.0
wcwidth==0.2.6
WeasyPrint==52.1
webargs==8.2.0
webencodings==0.5.1
Werkzeug==1.0.1
wrapt==1.15.0
WTForms==3.0.1
xlrd==2.0.1
xlwt==1.3.0
xmlschema==2.1.1
xmlsec==1.3.13
xmltodict==0.13.0
xmlunittest==0.5.0
xsdata==22.9
xxhash==3.2.0
yarl==1.8.2
zeep==4.2.1
zipp==3.15.0
</details>

### How can we reproduce your problem?

With `DD_APPSEC_ENABLED` on a Flask app, call an API with a UUID in the URL.

For example `GET /api/users/bacaedfa-6627-4377-ae53-720c3e16a79b` or `/api/companies/d48803f6-2a11-4b69-8e72-4c8c8bcb2db0/employees`

### What is the result that you get?

A warning coming from ddtrace:

`DDWAF object init called with unknown data structure: <class 'uuid.UUID'>, 1 additional messages skipped`.

### What is the result that you expected?

Not to see this warning. 



Upstream issue localstack/localstack#5395 closed
Upstream issue localstack/localstack#5395 referenced in the file:

  - [tests/contrib/botocore/test.py:1396](https://github.com/DataDog/dd-trace-py/blob/1.x/tests/contrib/botocore/test.py#L1396)

has been closed.

The code referencing this issue could potentially be updated.
    


Discussion:

@hghotra It looks like this upstream issue has been closed. Can you confirm if we can update `localstack` as you mentioned once this issue was resolved? Thanks!

Resolving patch:
diff --git a/ddtrace/_monkey.py b/ddtrace/_monkey.py
--- a/ddtrace/_monkey.py
+++ b/ddtrace/_monkey.py
@@ -16,6 +16,7 @@
     from typing import Any
     from typing import Callable
     from typing import List
+    from typing import Union
 
 
 log = get_logger(__name__)
@@ -145,8 +146,8 @@ class ModuleNotFoundException(PatchException):
     pass
 
 
-def _on_import_factory(module, prefix="ddtrace.contrib", raise_errors=True):
-    # type: (str, str, bool) -> Callable[[Any], None]
+def _on_import_factory(module, prefix="ddtrace.contrib", raise_errors=True, patch_indicator=True):
+    # type: (str, str, bool, Union[bool, List[str]]) -> Callable[[Any], None]
     """Factory to create an import hook for the provided module name"""
 
     def on_import(hook):
@@ -160,6 +161,8 @@ def on_import(hook):
             log.error("failed to import ddtrace module %r when patching on import", path, exc_info=True)
         else:
             imported_module.patch()
+            if hasattr(imported_module, "patch_submodules"):
+                imported_module.patch_submodules(patch_indicator)
             telemetry_lifecycle_writer.add_integration(module, PATCH_MODULES.get(module) is True)
 
     return on_import
@@ -214,7 +217,7 @@ def patch_iast(**patch_modules):
 
 
 def patch(raise_errors=True, patch_modules_prefix=DEFAULT_MODULES_PREFIX, **patch_modules):
-    # type: (bool, str, bool) -> None
+    # type: (bool, str, Union[List[str], bool]) -> None
     """Patch only a set of given modules.
 
     :param bool raise_errors: Raise error if one patch fail.
@@ -222,8 +225,8 @@ def patch(raise_errors=True, patch_modules_prefix=DEFAULT_MODULES_PREFIX, **patc
 
         >>> patch(psycopg=True, elasticsearch=True)
     """
-    contribs = [c for c, should_patch in patch_modules.items() if should_patch]
-    for contrib in contribs:
+    contribs = {c: patch_indicator for c, patch_indicator in patch_modules.items() if patch_indicator}
+    for contrib, patch_indicator in contribs.items():
         # Check if we have the requested contrib.
         if not os.path.isfile(os.path.join(os.path.dirname(__file__), "contrib", contrib, "__init__.py")):
             if raise_errors:
@@ -234,7 +237,9 @@ def patch(raise_errors=True, patch_modules_prefix=DEFAULT_MODULES_PREFIX, **patc
         modules_to_patch = _MODULES_FOR_CONTRIB.get(contrib, (contrib,))
         for module in modules_to_patch:
             # Use factory to create handler to close over `module` and `raise_errors` values from this loop
-            when_imported(module)(_on_import_factory(contrib, raise_errors=raise_errors))
+            when_imported(module)(
+                _on_import_factory(contrib, raise_errors=raise_errors, patch_indicator=patch_indicator)
+            )
 
         # manually add module to patched modules
         with _LOCK:
diff --git a/ddtrace/contrib/botocore/__init__.py b/ddtrace/contrib/botocore/__init__.py
--- a/ddtrace/contrib/botocore/__init__.py
+++ b/ddtrace/contrib/botocore/__init__.py
@@ -13,6 +13,11 @@
     from ddtrace import patch
     patch(botocore=True)
 
+To patch only specific botocore modules, pass a list of the module names instead::
+
+    from ddtrace import patch
+    patch(botocore=['s3', 'sns'])
+
 Configuration
 ~~~~~~~~~~~~~
 
@@ -97,5 +102,6 @@
 with require_modules(required_modules) as missing_modules:
     if not missing_modules:
         from .patch import patch
+        from .patch import patch_submodules
 
-        __all__ = ["patch"]
+        __all__ = ["patch", "patch_submodules"]
diff --git a/ddtrace/contrib/botocore/patch.py b/ddtrace/contrib/botocore/patch.py
--- a/ddtrace/contrib/botocore/patch.py
+++ b/ddtrace/contrib/botocore/patch.py
@@ -10,7 +10,9 @@
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Set
 from typing import Tuple
+from typing import Union
 
 import botocore.client
 import botocore.exceptions
@@ -37,6 +39,8 @@
 from ..trace_utils import unwrap
 
 
+_PATCHED_SUBMODULES = set()  # type: Set[str]
+
 if typing.TYPE_CHECKING:  # pragma: no cover
     from ddtrace import Span
 
@@ -319,14 +323,25 @@ def patch():
 
     wrapt.wrap_function_wrapper("botocore.client", "BaseClient._make_api_call", patched_api_call)
     Pin(service="aws").onto(botocore.client.BaseClient)
+    _PATCHED_SUBMODULES.clear()
 
 
 def unpatch():
+    _PATCHED_SUBMODULES.clear()
     if getattr(botocore.client, "_datadog_patch", False):
         setattr(botocore.client, "_datadog_patch", False)
         unwrap(botocore.client.BaseClient, "_make_api_call")
 
 
+def patch_submodules(submodules):
+    # type: (Union[List[str], bool]) -> None
+    if isinstance(submodules, bool) and submodules:
+        _PATCHED_SUBMODULES.clear()
+    elif isinstance(submodules, list):
+        submodules = [sub_module.lower() for sub_module in submodules]
+        _PATCHED_SUBMODULES.update(submodules)
+
+
 def patched_api_call(original_func, instance, args, kwargs):
 
     pin = Pin.get_from(instance)
@@ -335,6 +350,9 @@ def patched_api_call(original_func, instance, args, kwargs):
 
     endpoint_name = deep_getattr(instance, "_endpoint._endpoint_prefix")
 
+    if _PATCHED_SUBMODULES and endpoint_name not in _PATCHED_SUBMODULES:
+        return original_func(*args, **kwargs)
+
     with pin.tracer.trace(
         "{}.command".format(endpoint_name), service="{}.{}".format(pin.service, endpoint_name), span_type=SpanTypes.HTTP
     ) as span:


Test patch:
diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py
--- a/tests/contrib/botocore/test.py
+++ b/tests/contrib/botocore/test.py
@@ -31,6 +31,7 @@
 from ddtrace.constants import ERROR_STACK
 from ddtrace.constants import ERROR_TYPE
 from ddtrace.contrib.botocore.patch import patch
+from ddtrace.contrib.botocore.patch import patch_submodules
 from ddtrace.contrib.botocore.patch import unpatch
 from ddtrace.internal.compat import PY2
 from ddtrace.internal.compat import PYTHON_VERSION_INFO
@@ -67,6 +68,7 @@ class BotocoreTest(TracerTestCase):
 
     def setUp(self):
         patch()
+        patch_submodules(True)
 
         self.session = botocore.session.get_session()
         self.session.set_credentials(access_key="access-key", secret_key="secret-key")
@@ -78,6 +80,27 @@ def tearDown(self):
 
         unpatch()
 
+    @mock_ec2
+    @mock_s3
+    def test_patch_submodules(self):
+        patch_submodules(["s3"])
+        ec2 = self.session.create_client("ec2", region_name="us-west-2")
+        Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2)
+
+        ec2.describe_instances()
+
+        spans = self.get_spans()
+        assert spans == []
+
+        s3 = self.session.create_client("s3", region_name="us-west-2")
+        Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3)
+
+        s3.list_buckets()
+        s3.list_buckets()
+
+        spans = self.get_spans()
+        assert spans
+
     @mock_ec2
     def test_traced_client(self):
         ec2 = self.session.create_client("ec2", region_name="us-west-2")