Repository: honeycombio/beeline-python

Issue:
update for e&s
**Is your feature request related to a problem? Please describe.**

Environments & Services are coming to Honeycomb. The beeline should be updated to support sending data to environments while maintaining backward compatibility. To achieve this, we will set dataset from service name, and provide appropriate warnings depending on environment vs. non-environment teams.

**Describe the solution you'd like**

Tracing scenario for environment-aware teams:

* Dataset should no longer propagate across services for new environment key.
* `ServiceName` "required"†. If unset, will warn and set `service_name` AND `service.name` to `unknown_service:<process>` or `unknown_service:language`. Service name is used to populate dataset name; if `unknown_service.*` then truncate to`unknown_service` for dataset name. `service.name` field will be added to the already added `service_name` for closer consistency with OTel.
* `WriteKey` required. If unset, will warn.
* `Dataset` **ignored**. If unset, no warning (because no longer used). If set, will warn to clarify that it will be ignored in favor of service name.

Tracing scenario for legacy (non-environment-aware) teams:

* `ServiceName` "required"†. If unset, will warn.
* `WriteKey` required. If unset, will warn.
* `Dataset` required. If unset, will warn and default to `beeline-<language>`.

† While not technically required, service name is highly encouraged to avoid `unknown_service` and to instead properly describe the data being sent to Honeycomb (and the data being viewed in Honeycomb UI).

This will be addressed in a separate issue/PR: Set [spanKind value based on OTel spec](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind) for consistency across beelines and OTel.

### Notes from decision around defaulting when no service name is provided:
*Given* a beeline configured with a v2 api key and without a service name
*When* it sends HNY events
*Then* it should use `unknown_service` as the dataset
*And* it should use `unknown_service:process-name` as service_name

When given a legacy key, the behavior remains the same as today.

[Notes on trimming whitespace](https://honeycomb.quip.com/qdrvA6a9AHEI/notes-trimming-whitespace-ingest)

Comments:

Patch:
diff --git a/beeline/__init__.py b/beeline/__init__.py
index ed238fc..edabdee 100644
--- a/beeline/__init__.py
+++ b/beeline/__init__.py
@@ -10,6 +10,7 @@ from beeline.trace import SynchronousTracer
 from beeline.version import VERSION
 from beeline import internal
 import beeline.propagation.default
+import beeline.propagation
 import sys
 # pyflakes
 assert internal
@@ -74,15 +75,68 @@ class Beeline(object):
         if debug:
             self._init_logger()
 
+        def IsClassicKey(writekey):
+            return len(writekey) == 32
+
         # allow setting some values from the environment
         if not writekey:
             writekey = os.environ.get('HONEYCOMB_WRITEKEY', '')
+        # also check API_KEY just in case
+        if not writekey:
+            writekey = os.environ.get('HONEYCOMB_API_KEY', '')
+        if not writekey:
+            logging.error(
+                'writekey not set! set the writekey if you want to send data to honeycomb'
+            )
+
+        if IsClassicKey(writekey):
+            if not dataset:
+                dataset = os.environ.get('HONEYCOMB_DATASET', '')
+            if not dataset:
+                logging.error(
+                    'dataset not set! set a value for dataset if you want to send data to honeycomb'
+                )
+        else:
+            if dataset:
+                logging.error(
+                    'dataset will be ignored in favor of service name'
+                )
 
-        if not dataset:
-            dataset = os.environ.get('HONEYCOMB_DATASET', '')
-
+        default_service_name = "unknown_service"
+        process_name = os.environ.get('PROCESS_EXECUTABLE_NAME', '')
         if not service_name:
-            service_name = os.environ.get('HONEYCOMB_SERVICE', dataset)
+            service_name = os.environ.get('HONEYCOMB_SERVICE')
+        # also check SERVICE_NAME just in case
+        if not service_name:
+            service_name = os.environ.get('SERVICE_NAME', '')
+        # no service name, set default
+        if not service_name:
+            service_name = default_service_name
+            if process_name:
+                service_name += ":" + process_name
+            else:
+                service_name += ":python"
+            logging.error(
+                'service name not set! service name will be ' + service_name
+            )
+            if not IsClassicKey(writekey):
+                logging.error(
+                    'data will be sent to unknown_service'
+                )
+
+        if not IsClassicKey(writekey):
+            # set dataset based on service name
+            dataset = service_name
+            if dataset.strip() != dataset:
+                # whitespace detected. trim whitespace, warn on diff
+                logging.error(
+                    'service name has unexpected spaces'
+                )
+                dataset = service_name.strip()
+
+            # set default, truncate to unknown_service if needed
+            if dataset == "" or dataset.startswith("unknown_service"):
+                dataset = "unknown_service"
 
         self.client = Client(
             writekey=writekey, dataset=dataset, sample_rate=sample_rate,
@@ -94,16 +148,16 @@ class Beeline(object):
             debug=debug,
         )
 
+        if IsClassicKey(writekey):
+            beeline.propagation.propagate_dataset = True
+        else:
+            beeline.propagation.propagate_dataset = False
+
         self.log('initialized honeycomb client: writekey=%s dataset=%s service_name=%s',
                  writekey, dataset, service_name)
-        if not writekey:
-            self.log(
-                'writekey not set! set the writekey if you want to send data to honeycomb')
-        if not dataset:
-            self.log(
-                'dataset not set! set a value for dataset if you want to send data to honeycomb')
 
         self.client.add_field('service_name', service_name)
+        self.client.add_field('service.name', service_name)
         self.client.add_field('meta.beeline_version', VERSION)
         self.client.add_field('meta.local_hostname', socket.gethostname())
 
diff --git a/beeline/propagation/__init__.py b/beeline/propagation/__init__.py
index 7b9c1d9..0e3a7ca 100644
--- a/beeline/propagation/__init__.py
+++ b/beeline/propagation/__init__.py
@@ -3,6 +3,10 @@ from abc import ABCMeta, abstractmethod
 import beeline
 
 
+def propagate_dataset():
+    return bool
+
+
 class PropagationContext(object):
     '''
     PropagationContext represents information that can either be read from, or
diff --git a/beeline/propagation/honeycomb.py b/beeline/propagation/honeycomb.py
index 6957a0a..04db12f 100644
--- a/beeline/propagation/honeycomb.py
+++ b/beeline/propagation/honeycomb.py
@@ -51,7 +51,7 @@ def marshal_propagation_context(propagation_context):
                   "parent_id={}".format(propagation_context.parent_id),
                   "context={}".format(trace_fields)]
 
-    if propagation_context.dataset:
+    if beeline.propagation.propagate_dataset and propagation_context.dataset:
         components.insert(0, "dataset={}".format(quote(propagation_context.dataset)))
 
     trace_header = "{};{}".format(version, ",".join(components))
@@ -94,7 +94,7 @@ def unmarshal_propagation_context_with_dataset(trace_header):
             parent_id = v
         elif k == 'context':
             context = json.loads(base64.b64decode(v.encode()).decode())
-        elif k == 'dataset':
+        elif k == 'dataset' and beeline.propagation.propagate_dataset:
             dataset = unquote(v)
 
     # context should be a dict
diff --git a/examples/hello-world/README.md b/examples/hello-world/README.md
new file mode 100644
index 0000000..95572b6
--- /dev/null
+++ b/examples/hello-world/README.md
@@ -0,0 +1,9 @@
+# hello world example
+
+## Setup and Run
+
+1. Set environment variable for `HONEYCOMB_API_KEY`
+1. In top-level directory of repo, run `poetry build` to create a wheel package in the `dist` directory
+1. In hello-world directory, ensure version of beeline in `pyproject.toml` matches the wheel package
+1. In hello-world directory, run `poetry install`
+1. In hello-world directory, run `poetry run python3 app.py`
diff --git a/examples/hello-world/app.py b/examples/hello-world/app.py
new file mode 100644
index 0000000..09f8351
--- /dev/null
+++ b/examples/hello-world/app.py
@@ -0,0 +1,21 @@
+import beeline
+import os
+
+beeline.init(
+    # Get this via https://ui.honeycomb.io/account after signing up for Honeycomb
+    writekey=os.environ.get('HONEYCOMB_API_KEY'),
+    api_host=os.environ.get('HONEYCOMB_API_ENDPOINT', 'http://api.honeycomb.io:443'),
+    # The name of your app is a good choice to start with
+    # dataset='my-python-beeline-app', # only needed for classic
+    service_name=os.environ.get('SERVICE_NAME', 'my-python-app'),
+    debug=True, # enable to see telemetry in console
+)
+
+@beeline.traced(name='hello_world')
+def hello_world():
+    print('hello world')
+    beeline.add_context_field('message', 'hello world')
+
+hello_world()
+
+beeline.close()
\ No newline at end of file
diff --git a/examples/hello-world/pyproject.toml b/examples/hello-world/pyproject.toml
new file mode 100644
index 0000000..7c7ef9f
--- /dev/null
+++ b/examples/hello-world/pyproject.toml
@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "hello-world"
+version = "0.1.0"
+description = "print hello world"
+authors = ["honeycombio"]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+honeycomb-beeline = {path = "../../dist/honeycomb_beeline-3.42.99-py3-none-any.whl", develop = false}
+# honeycomb-beeline = "^3.2.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"


Test patch:
diff --git a/beeline/propagation/test_honeycomb.py b/beeline/propagation/test_honeycomb.py
index 23f321a..d5ddd43 100644
--- a/beeline/propagation/test_honeycomb.py
+++ b/beeline/propagation/test_honeycomb.py
@@ -1,4 +1,5 @@
 import unittest
+import beeline.propagation
 from beeline.propagation import DictRequest, PropagationContext
 import beeline.propagation.honeycomb as hc
 
@@ -21,6 +22,7 @@ class TestMarshalUnmarshal(unittest.TestCase):
 
     def test_roundtrip_with_dataset(self):
         '''Verify that we can successfully roundtrip (marshal and unmarshal)'''
+        beeline.propagation.propagate_dataset = True
         dataset = "blorp blorp"
         trace_id = "bloop"
         parent_id = "scoop"
@@ -34,6 +36,22 @@ class TestMarshalUnmarshal(unittest.TestCase):
         self.assertEqual(parent_id, new_parent_id)
         self.assertEqual(trace_fields, new_trace_fields)
 
+    def test_roundtrip_with_dataset_propagation_disabled(self):
+        '''Verify that we can successfully roundtrip (marshal and unmarshal) without dataset propagation'''
+        beeline.propagation.propagate_dataset = False
+        dataset = "blorp blorp"
+        trace_id = "bloop"
+        parent_id = "scoop"
+        trace_fields = {"key": "value"}
+        pc = PropagationContext(trace_id, parent_id, trace_fields, dataset)
+        header = hc.marshal_propagation_context(pc)
+        new_trace_id, new_parent_id, new_trace_fields, new_dataset = hc.unmarshal_propagation_context_with_dataset(
+            header)
+        self.assertIsNone(new_dataset)
+        self.assertEqual(trace_id, new_trace_id)
+        self.assertEqual(parent_id, new_parent_id)
+        self.assertEqual(trace_fields, new_trace_fields)
+
 
 class TestHoneycombHTTPTraceParserHook(unittest.TestCase):
     def test_has_header(self):
@@ -55,6 +73,7 @@ class TestHoneycombHTTPTraceParserHook(unittest.TestCase):
 
 class TestHoneycombHTTPTracePropagationHook(unittest.TestCase):
     def test_generates_correct_header(self):
+        beeline.propagation.propagate_dataset = True
         dataset = "blorp blorp"
         trace_id = "bloop"
         parent_id = "scoop"
@@ -65,3 +84,16 @@ class TestHoneycombHTTPTracePropagationHook(unittest.TestCase):
         self.assertIn('X-Honeycomb-Trace', headers)
         self.assertEqual(headers['X-Honeycomb-Trace'],
                          "1;dataset=blorp%20blorp,trace_id=bloop,parent_id=scoop,context=eyJrZXkiOiAidmFsdWUifQ==")
+
+    def test_generates_correct_header_with_dataset_propagation_disabled(self):
+        beeline.propagation.propagate_dataset = False
+        dataset = "blorp blorp"
+        trace_id = "bloop"
+        parent_id = "scoop"
+        trace_fields = {"key": "value"}
+        pc = PropagationContext(
+            trace_id, parent_id, trace_fields, dataset)
+        headers = hc.http_trace_propagation_hook(pc)
+        self.assertIn('X-Honeycomb-Trace', headers)
+        self.assertEqual(headers['X-Honeycomb-Trace'],
+                         "1;trace_id=bloop,parent_id=scoop,context=eyJrZXkiOiAidmFsdWUifQ==")
diff --git a/beeline/test_trace.py b/beeline/test_trace.py
index 5033618..84ef4d1 100644
--- a/beeline/test_trace.py
+++ b/beeline/test_trace.py
@@ -7,6 +7,7 @@ import unittest
 import re
 import os
 import binascii
+import beeline.propagation
 
 from libhoney import Event
 
@@ -624,6 +625,7 @@ class TestSpan(unittest.TestCase):
 
 class TestPropagationHooks(unittest.TestCase):
     def test_propagate_and_start_trace_uses_honeycomb_header(self):
+        beeline.propagation.propagate_dataset = True
         # FIXME: Test basics, including error handling and custom hooks
 
         # implicitly tests finish_span
@@ -653,6 +655,40 @@ class TestPropagationHooks(unittest.TestCase):
         # ensure that there is no current trace
         self.assertIsNone(tracer._trace)
 
+    def test_propagate_and_start_trace_uses_honeycomb_header_with_dataset_propagation_disabled(self):
+        beeline.propagation.propagate_dataset = False
+        # FIXME: Test basics, including error handling and custom hooks
+
+        # implicitly tests finish_span
+        m_client = Mock()
+        # these values are used before sending
+        m_client.new_event.return_value.start_time = datetime.datetime.now()
+        m_client.new_event.return_value.sample_rate = 1
+        tracer = SynchronousTracer(m_client)
+
+        header_value = '1;dataset=flibble,trace_id=bloop,parent_id=scoop,context=e30K'
+        req = DictRequest({
+            # case shouldn't matter
+            'X-HoNEyComb-TrACE': header_value,
+        })
+
+        span = tracer.propagate_and_start_trace(
+            context={'big': 'important_stuff'}, request=req)
+        self.assertEqual(tracer._trace.stack[0], span)
+        self.assertEqual(span.trace_id, "bloop")
+        self.assertEqual(span.parent_id, "scoop")
+
+        tracer.finish_trace(span)
+
+        # testing the absence of dataset
+        self.assertNotEqual(getattr(span.event, 'dataset'), "flibble")
+        self.assertNotIsInstance(span.event.dataset, str)
+
+        # ensure the event is sent
+        span.event.send_presampled.assert_called_once_with()
+        # ensure that there is no current trace
+        self.assertIsNone(tracer._trace)
+
     def test_propagate_and_start_trace_uses_w3c_header_as_fallback(self):
         # implicitly tests finish_span
         m_client = Mock()