Repository: elastic/apm-agent-python

Issue:
Remove query string sanitization in 6.0
To align with other agents, we should remove query string sanitization in the next major version. 

See https://github.com/elastic/apm/pull/334


Discussion:

Resolving patch:
diff --git a/elasticapm/conf/__init__.py b/elasticapm/conf/__init__.py
--- a/elasticapm/conf/__init__.py
+++ b/elasticapm/conf/__init__.py
@@ -503,11 +503,12 @@ class Config(_ConfigBase):
             "elasticapm.processors.sanitize_http_response_cookies",
             "elasticapm.processors.sanitize_http_headers",
             "elasticapm.processors.sanitize_http_wsgi_env",
-            "elasticapm.processors.sanitize_http_request_querystring",
             "elasticapm.processors.sanitize_http_request_body",
         ],
     )
-    sanitize_field_names = _ListConfigValue("SANITIZE_FIELD_NAMES", default=BASE_SANITIZE_FIELD_NAMES)
+    sanitize_field_names = _ListConfigValue(
+        "SANITIZE_FIELD_NAMES", type=starmatch_to_regex, default=BASE_SANITIZE_FIELD_NAMES
+    )
     metrics_sets = _ListConfigValue(
         "METRICS_SETS",
         default=[
diff --git a/elasticapm/conf/constants.py b/elasticapm/conf/constants.py
--- a/elasticapm/conf/constants.py
+++ b/elasticapm/conf/constants.py
@@ -32,6 +32,31 @@
 import re
 from collections import namedtuple
 
+
+def _starmatch_to_regex(pattern):
+    """
+    This is a duplicate of starmatch_to_regex() in utils/__init__.py
+
+    Duplication to avoid circular imports
+    """
+    options = re.DOTALL
+    # check if we are case sensitive
+    if pattern.startswith("(?-i)"):
+        pattern = pattern[5:]
+    else:
+        options |= re.IGNORECASE
+    i, n = 0, len(pattern)
+    res = []
+    while i < n:
+        c = pattern[i]
+        i = i + 1
+        if c == "*":
+            res.append(".*")
+        else:
+            res.append(re.escape(c))
+    return re.compile(r"(?:%s)\Z" % "".join(res), options)
+
+
 EVENTS_API_PATH = "intake/v2/events"
 AGENT_CONFIG_PATH = "config/v1/agents"
 
@@ -46,7 +71,7 @@
 
 HTTP_WITH_BODY = {"POST", "PUT", "PATCH", "DELETE"}
 
-MASK = 8 * "*"
+MASK = "[REDACTED]"
 
 EXCEPTION_CHAIN_MAX_DEPTH = 50
 
@@ -59,17 +84,22 @@
 
 HARDCODED_PROCESSORS = ["elasticapm.processors.add_context_lines_to_frames"]
 
-BASE_SANITIZE_FIELD_NAMES = [
-    "authorization",
+BASE_SANITIZE_FIELD_NAMES_UNPROCESSED = [
     "password",
-    "secret",
     "passwd",
-    "token",
-    "api_key",
-    "access_token",
-    "sessionid",
+    "pwd",
+    "secret",
+    "*key",
+    "*token*",
+    "*session*",
+    "*credit*",
+    "*card*",
+    "authorization",
+    "set-cookie",
 ]
 
+BASE_SANITIZE_FIELD_NAMES = [_starmatch_to_regex(x) for x in BASE_SANITIZE_FIELD_NAMES_UNPROCESSED]
+
 OUTCOME = namedtuple("OUTCOME", ["SUCCESS", "FAILURE", "UNKNOWN"])(
     SUCCESS="success", FAILURE="failure", UNKNOWN="unknown"
 )
diff --git a/elasticapm/processors.py b/elasticapm/processors.py
--- a/elasticapm/processors.py
+++ b/elasticapm/processors.py
@@ -29,17 +29,14 @@
 #  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 
 
-import re
 import warnings
 from collections import defaultdict
 
 from elasticapm.conf.constants import BASE_SANITIZE_FIELD_NAMES, ERROR, MASK, SPAN, TRANSACTION
 from elasticapm.utils import compat, varmap
-from elasticapm.utils.encoding import force_text, keyword_field
+from elasticapm.utils.encoding import force_text
 from elasticapm.utils.stacks import get_lines_from_file
 
-SANITIZE_VALUE_PATTERNS = [re.compile(r"^[- \d]{16,19}$")]  # credit card numbers, with or without spacers
-
 
 def for_events(*events):
     """
@@ -116,7 +113,7 @@ def sanitize_http_request_cookies(client, event):
 
     # sanitize request.header.cookie string
     try:
-        cookie_string = event["context"]["request"]["headers"]["cookie"]
+        cookie_string = force_text(event["context"]["request"]["headers"]["cookie"], errors="replace")
         event["context"]["request"]["headers"]["cookie"] = _sanitize_string(
             cookie_string, "; ", "=", sanitize_field_names=client.config.sanitize_field_names
         )
@@ -134,7 +131,7 @@ def sanitize_http_response_cookies(client, event):
     :return: The modified event
     """
     try:
-        cookie_string = event["context"]["response"]["headers"]["set-cookie"]
+        cookie_string = force_text(event["context"]["response"]["headers"]["set-cookie"], errors="replace")
         event["context"]["response"]["headers"]["set-cookie"] = _sanitize_string(
             cookie_string, ";", "=", sanitize_field_names=client.config.sanitize_field_names
         )
@@ -190,32 +187,6 @@ def sanitize_http_wsgi_env(client, event):
     return event
 
 
-@for_events(ERROR, TRANSACTION)
-def sanitize_http_request_querystring(client, event):
-    """
-    Sanitizes http request query string
-    :param client: an ElasticAPM client
-    :param event: a transaction or error event
-    :return: The modified event
-    """
-    try:
-        query_string = force_text(event["context"]["request"]["url"]["search"], errors="replace")
-    except (KeyError, TypeError):
-        return event
-    if "=" in query_string:
-        sanitized_query_string = _sanitize_string(
-            query_string, "&", "=", sanitize_field_names=client.config.sanitize_field_names
-        )
-        full_url = event["context"]["request"]["url"]["full"]
-        # we need to pipe the sanitized string through encoding.keyword_field to ensure that the maximum
-        # length of keyword fields is still ensured.
-        event["context"]["request"]["url"]["search"] = keyword_field(sanitized_query_string)
-        event["context"]["request"]["url"]["full"] = keyword_field(
-            full_url.replace(query_string, sanitized_query_string)
-        )
-    return event
-
-
 @for_events(ERROR, TRANSACTION)
 def sanitize_http_request_body(client, event):
     """
@@ -276,16 +247,13 @@ def mark_in_app_frames(client, event):
 
 def _sanitize(key, value, **kwargs):
     if "sanitize_field_names" in kwargs:
-        sanitize_field_names = frozenset(kwargs["sanitize_field_names"])
+        sanitize_field_names = kwargs["sanitize_field_names"]
     else:
-        sanitize_field_names = frozenset(BASE_SANITIZE_FIELD_NAMES)
+        sanitize_field_names = BASE_SANITIZE_FIELD_NAMES
 
     if value is None:
         return
 
-    if isinstance(value, compat.string_types) and any(pattern.match(value) for pattern in SANITIZE_VALUE_PATTERNS):
-        return MASK
-
     if isinstance(value, dict):
         # varmap will call _sanitize on each k:v pair of the dict, so we don't
         # have to do anything with dicts here
@@ -296,7 +264,7 @@ def _sanitize(key, value, **kwargs):
 
     key = key.lower()
     for field in sanitize_field_names:
-        if field in key:
+        if field.match(key.strip()):
             # store mask as a fixed length for security
             return MASK
     return value


Test patch:
diff --git a/tests/processors/tests.py b/tests/processors/tests.py
--- a/tests/processors/tests.py
+++ b/tests/processors/tests.py
@@ -40,7 +40,7 @@
 
 import elasticapm
 from elasticapm import Client, processors
-from elasticapm.conf.constants import BASE_SANITIZE_FIELD_NAMES, ERROR, SPAN, TRANSACTION
+from elasticapm.conf.constants import BASE_SANITIZE_FIELD_NAMES_UNPROCESSED, ERROR, SPAN, TRANSACTION
 from elasticapm.utils import compat
 
 
@@ -49,33 +49,31 @@ def http_test_data():
     return {
         "context": {
             "request": {
-                "body": "foo=bar&password=123456&the_secret=abc&cc=1234567890098765&custom_field=123",
+                "body": "foo=bar&password=123456&secret=abc&cc=1234567890098765&custom_field=123",
                 "env": {
                     "foo": "bar",
                     "password": "hello",
-                    "the_secret": "hello",
-                    "a_password_here": "hello",
+                    "secret": "hello",
                     "custom_env": "bye",
                 },
                 "headers": {
                     "foo": "bar",
                     "password": "hello",
-                    "the_secret": "hello",
-                    "a_password_here": "hello",
+                    "secret": "hello",
                     "authorization": "bearer xyz",
                     "some-header": "some-secret-value",
+                    "cookie": "foo=bar; baz=foo",
                 },
                 "cookies": {
                     "foo": "bar",
                     "password": "topsecret",
-                    "the_secret": "topsecret",
+                    "secret": "topsecret",
                     "sessionid": "123",
-                    "a_password_here": "123456",
                     "custom-sensitive-cookie": "123",
                 },
                 "url": {
-                    "full": "http://example.com/bla?foo=bar&password=123456&the_secret=abc&cc=1234567890098765&custom_field=123",
-                    "search": "foo=bar&password=123456&the_secret=abc&cc=1234567890098765&custom_field=123",
+                    "full": "http://example.com/bla?foo=bar&password=123456&secret=abc&cc=1234567890098765&custom_field=123",
+                    "search": "foo=bar&password=123456&secret=abc&cc=1234567890098765&custom_field=123",
                 },
             },
             "response": {
@@ -83,10 +81,10 @@ def http_test_data():
                 "headers": {
                     "foo": "bar",
                     "password": "hello",
-                    "the_secret": "hello",
-                    "a_password_here": "hello",
+                    "secret": "hello",
                     "authorization": "bearer xyz",
                     "some-header": "some-secret-value",
+                    "cookie": "foo=bar; baz=foo",
                 },
             },
         }
@@ -96,7 +94,10 @@ def http_test_data():
 @pytest.mark.parametrize(
     "elasticapm_client, custom_field",
     [
-        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["sensitive-stacktrace-val"]}, processors.MASK),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["sensitive-stacktrace-val"]},
+            processors.MASK,
+        ),
         ({}, "bye"),
     ],
     indirect=["elasticapm_client"],
@@ -109,8 +110,7 @@ def test_stacktrace(elasticapm_client, custom_field):
                     "vars": {
                         "foo": "bar",
                         "password": "hello",
-                        "the_secret": "hello",
-                        "a_password_here": "hello",
+                        "secret": "hello",
                         "sensitive-stacktrace-val": "bye",
                     }
                 }
@@ -122,8 +122,7 @@ def test_stacktrace(elasticapm_client, custom_field):
                             "vars": {
                                 "foo": "bar",
                                 "password": "hello",
-                                "the_secret": "hello",
-                                "a_password_here": "hello",
+                                "secret": "hello",
                                 "sensitive-stacktrace-val": "bye",
                             }
                         }
@@ -135,8 +134,7 @@ def test_stacktrace(elasticapm_client, custom_field):
                                     "vars": {
                                         "foo": "bar",
                                         "password": "hello",
-                                        "the_secret": "hello",
-                                        "a_password_here": "hello",
+                                        "secret": "hello",
                                         "sensitive-stacktrace-val": "bye",
                                     }
                                 }
@@ -164,10 +162,8 @@ def test_stacktrace(elasticapm_client, custom_field):
         assert vars["foo"] == "bar"
         assert "password" in vars
         assert vars["password"] == processors.MASK
-        assert "the_secret" in vars
-        assert vars["the_secret"] == processors.MASK
-        assert "a_password_here" in vars
-        assert vars["a_password_here"] == processors.MASK
+        assert "secret" in vars
+        assert vars["secret"] == processors.MASK
         assert "sensitive-stacktrace-val" in vars
         assert vars["sensitive-stacktrace-val"] == custom_field
 
@@ -184,14 +180,24 @@ def test_remove_http_request_body(http_test_data):
     "elasticapm_client, custom_field, expected_header_cookies",
     [
         (
-            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["custom-sensitive-cookie"]},
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["custom-sensitive-cookie"]},
             {"custom-sensitive-cookie": processors.MASK},
-            "foo=bar; password={0}; the_secret={0}; csrftoken={0}; custom-sensitive-cookie={0}".format(processors.MASK),
+            "foo=bar; password={0}; secret={0}; csrftoken={0}; custom-sensitive-cookie={0}".format(processors.MASK),
         ),
         (
             {},
             {"custom-sensitive-cookie": "123"},
-            "foo=bar; password={0}; the_secret={0}; csrftoken={0}; custom-sensitive-cookie=123".format(processors.MASK),
+            "foo=bar; password={0}; secret={0}; csrftoken={0}; custom-sensitive-cookie=123".format(processors.MASK),
+        ),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["custom-sensitive-*"]},
+            {"custom-sensitive-cookie": processors.MASK},
+            "foo=bar; password={0}; secret={0}; csrftoken={0}; custom-sensitive-cookie={0}".format(processors.MASK),
+        ),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["custom-sensitive"]},
+            {"custom-sensitive-cookie": "123"},
+            "foo=bar; password={0}; secret={0}; csrftoken={0}; custom-sensitive-cookie=123".format(processors.MASK),
         ),
     ],
     indirect=["elasticapm_client"],
@@ -199,15 +205,14 @@ def test_remove_http_request_body(http_test_data):
 def test_sanitize_http_request_cookies(elasticapm_client, custom_field, expected_header_cookies, http_test_data):
     http_test_data["context"]["request"]["headers"][
         "cookie"
-    ] = "foo=bar; password=12345; the_secret=12345; csrftoken=abc; custom-sensitive-cookie=123"
+    ] = "foo=bar; password=12345; secret=12345; csrftoken=abc; custom-sensitive-cookie=123"
 
     result = processors.sanitize_http_request_cookies(elasticapm_client, http_test_data)
     expected = {
         "foo": "bar",
         "password": processors.MASK,
-        "the_secret": processors.MASK,
+        "secret": processors.MASK,
         "sessionid": processors.MASK,
-        "a_password_here": processors.MASK,
     }
 
     expected.update(custom_field)
@@ -221,7 +226,7 @@ def test_sanitize_http_request_cookies(elasticapm_client, custom_field, expected
     "elasticapm_client, expected_cookies",
     [
         (
-            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["custom-sensitive-cookie"]},
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["custom-sensitive-cookie"]},
             "foo=bar; httponly; secure ; sessionid={0}; httponly; secure; custom-sensitive-cookie={0}".format(
                 processors.MASK
             ),
@@ -248,20 +253,29 @@ def test_sanitize_http_response_cookies(elasticapm_client, expected_cookies, htt
 @pytest.mark.parametrize(
     "elasticapm_client, custom_header",
     [
-        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["some-header"]}, {"some-header": processors.MASK}),
-        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["some-"]}, {"some-header": processors.MASK}),
-        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["other-val"]}, {"some-header": "some-secret-value"}),
-        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["-"]}, {"some-header": processors.MASK}),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["some-header"]},
+            {"some-header": processors.MASK},
+        ),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["some-*"]},
+            {"some-header": processors.MASK},
+        ),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["other-val"]},
+            {"some-header": "some-secret-value"},
+        ),
+        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["*-*"]}, {"some-header": processors.MASK}),
     ],
     indirect=["elasticapm_client"],
 )
 def test_sanitize_http_headers(elasticapm_client, custom_header, http_test_data):
     result = processors.sanitize_http_headers(elasticapm_client, http_test_data)
     expected = {
+        "cookie": "foo=bar; baz=foo",
         "foo": "bar",
         "password": processors.MASK,
-        "the_secret": processors.MASK,
-        "a_password_here": processors.MASK,
+        "secret": processors.MASK,
         "authorization": processors.MASK,
     }
     expected.update(custom_header)
@@ -272,7 +286,10 @@ def test_sanitize_http_headers(elasticapm_client, custom_header, http_test_data)
 @pytest.mark.parametrize(
     "elasticapm_client, custom_env",
     [
-        ({"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["custom_env"]}, {"custom_env": processors.MASK}),
+        (
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["custom_env"]},
+            {"custom_env": processors.MASK},
+        ),
         ({}, {"custom_env": "bye"}),
     ],
     indirect=["elasticapm_client"],
@@ -282,8 +299,7 @@ def test_sanitize_http_wgi_env(elasticapm_client, custom_env, http_test_data):
     expected = {
         "foo": "bar",
         "password": processors.MASK,
-        "the_secret": processors.MASK,
-        "a_password_here": processors.MASK,
+        "secret": processors.MASK,
     }
     expected.update(custom_env)
     assert result["context"]["request"]["env"] == expected
@@ -293,37 +309,10 @@ def test_sanitize_http_wgi_env(elasticapm_client, custom_env, http_test_data):
     "elasticapm_client, expected",
     [
         (
-            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["custom_field"]},
-            "foo=bar&password={0}&the_secret={0}&cc={0}&custom_field={0}".format(processors.MASK),
+            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES_UNPROCESSED + ["custom_field"]},
+            "foo=bar&password={0}&secret={0}&cc=1234567890098765&custom_field={0}".format(processors.MASK),
         ),
-        ({}, "foo=bar&password={0}&the_secret={0}&cc={0}&custom_field=123".format(processors.MASK)),
-    ],
-    indirect=["elasticapm_client"],
-)
-def test_sanitize_http_query_string(elasticapm_client, expected, http_test_data):
-    result = processors.sanitize_http_request_querystring(elasticapm_client, http_test_data)
-    assert result["context"]["request"]["url"]["search"] == expected
-    assert result["context"]["request"]["url"]["full"].endswith(expected)
-
-
-def test_sanitize_http_query_string_max_length(elasticapm_client):
-    qs = "api_key=1&v=" + 1020 * "x"
-    data = {"context": {"request": {"url": {"full": "http://example.com/bla?" + qs, "search": qs}}}}
-    result = processors.sanitize_http_request_querystring(elasticapm_client, data)
-    for val in (result["context"]["request"]["url"]["search"], result["context"]["request"]["url"]["full"]):
-        assert "api_key=********" in val
-        assert len(val) == 1024
-        assert val.endswith(u"x…")
-
-
-@pytest.mark.parametrize(
-    "elasticapm_client, expected",
-    [
-        (
-            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["custom_field"]},
-            "foo=bar&password={0}&the_secret={0}&cc={0}&custom_field={0}".format(processors.MASK),
-        ),
-        ({}, "foo=bar&password={0}&the_secret={0}&cc={0}&custom_field=123".format(processors.MASK)),
+        ({}, "foo=bar&password={0}&secret={0}&cc=1234567890098765&custom_field=123".format(processors.MASK)),
     ],
     indirect=["elasticapm_client"],
 )
@@ -332,32 +321,6 @@ def test_post_as_string(elasticapm_client, expected, http_test_data):
     assert result["context"]["request"]["body"] == expected
 
 
-@pytest.mark.parametrize(
-    "elasticapm_client, expected",
-    [
-        (
-            {"sanitize_field_names": BASE_SANITIZE_FIELD_NAMES + ["custom_field"]},
-            "foo=bar&password={0}&the_secret={0}&cc={0}&custom_field={0}".format(processors.MASK),
-        ),
-        ({}, "foo=bar&password={0}&the_secret={0}&cc={0}&custom_field=123".format(processors.MASK)),
-    ],
-    indirect=["elasticapm_client"],
-)
-def test_querystring_as_string_with_partials(elasticapm_client, expected, http_test_data):
-    result = processors.sanitize_http_request_querystring(elasticapm_client, http_test_data)
-    assert result["context"]["request"]["url"]["search"] == expected
-
-
-def test_sanitize_credit_card():
-    result = processors._sanitize("foo", "4242424242424242")
-    assert result == processors.MASK
-
-
-def test_sanitize_credit_card_with_spaces():
-    result = processors._sanitize("foo", "4242 4242 4242 4242")
-    assert result == processors.MASK
-
-
 def test_sanitize_dict():
     result = processors._sanitize("foo", {1: 2})
     assert result == {1: 2}
@@ -365,19 +328,13 @@ def test_sanitize_dict():
 
 def test_non_utf8_encoding(elasticapm_client, http_test_data):
     broken = compat.b("broken=") + u"aéöüa".encode("latin-1")
-    http_test_data["context"]["request"]["url"]["search"] = broken
-    result = processors.sanitize_http_request_querystring(elasticapm_client, http_test_data)
-    assert result["context"]["request"]["url"]["search"] == u"broken=a\ufffd\ufffd\ufffda"
+    http_test_data["context"]["request"]["headers"]["cookie"] = broken
+    result = processors.sanitize_http_request_cookies(elasticapm_client, http_test_data)
+    assert result["context"]["request"]["headers"]["cookie"] == u"broken=a\ufffd\ufffd\ufffda"
 
 
 def test_remove_stacktrace_locals():
-    data = {
-        "exception": {
-            "stacktrace": [
-                {"vars": {"foo": "bar", "password": "hello", "the_secret": "hello", "a_password_here": "hello"}}
-            ]
-        }
-    }
+    data = {"exception": {"stacktrace": [{"vars": {"foo": "bar", "password": "hello", "secret": "hello"}}]}}
     result = processors.remove_stacktrace_locals(None, data)
 
     assert "stacktrace" in result["exception"]