Repository: honeycombio/refinery
Pull request #708 (2023-06-08 19:43:34)

feat: Add EMAThroughput sampler support
Depends on #707

## Which problem is this PR solving?

- Adds the new [EMA Throughput sampler](https://github.com/honeycombio/dynsampler-go/blob/main/emathroughput.go) to the set of supported samplers.

## Short description of the changes

- Add EMAThroughput sampler
- Add it to configuration and validation data
- Regen doc




Patch:
diff --git a/config/metadata/rulesMeta.yaml b/config/metadata/rulesMeta.yaml
index 28cc801..ed026b7 100644
--- a/config/metadata/rulesMeta.yaml
+++ b/config/metadata/rulesMeta.yaml
@@ -88,11 +88,12 @@ groups:
         type: int
         summary: is the maximum number of keys to track.
         description: >
-          MaxKeys, if greater than 0, limits the number of distinct keys tracked
-          by the sampler. Once MaxKeys is reached, new keys will not be included
-          in the sample rate map, but existing keys will continue to be be
-          counted. You can use this to keep the sample rate map size under
-          control.
+          MaxKeys limits the number of distinct keys tracked by the sampler.
+          Once MaxKeys is reached, new keys will not be included in the sample
+          rate map, but existing keys will continue to be be counted. You can
+          use this to keep the sample rate map size under control. Defaults to
+          500; dynamic samplers will rarely achieve their goals with more keys
+          than this.
       - name: UseTraceLength
         type: bool
         summary: indicates whether to include the trace length as part of the key.
@@ -232,11 +233,12 @@ groups:
         type: int
         summary: is the maximum number of keys to track.
         description: >
-          MaxKeys, if greater than 0, limits the number of distinct keys tracked
-          by the sampler. Once MaxKeys is reached, new keys will not be included
-          in the sample rate map, but existing keys will continue to be be
-          counted. You can use this to keep the sample rate map size under
-          control.
+          MaxKeys limits the number of distinct keys tracked by the sampler.
+          Once MaxKeys is reached, new keys will not be included in the sample
+          rate map, but existing keys will continue to be be counted. You can
+          use this to keep the sample rate map size under control. Defaults to
+          500; dynamic samplers will rarely achieve their goals with more keys
+          than this.
       - name: UseTraceLength
         type: bool
         summary: indicates whether to include the trace length as part of the key.
@@ -246,7 +248,159 @@ groups:
           are normally small variations in trace length you may want to leave
           this off. If traces are consistent lengths and changes in trace length
           is a useful indicator of traces you'd like to see in Honeycomb, set
-          this to true. Eligible for live reload.
+          this to true.
+
+  - name: EMAThroughputSampler
+    title: EMA Throughput Sampler
+    description: >
+      The Exponential Moving Average (EMA) Throughput Sampler attempts to
+      achieve a given throughput -- number of spans per second -- weighting rare
+      traffic and frequent traffic differently so as to end up with the correct
+      rate.
+
+      The EMAThroughputSampler is an improvement upon the TotalThroughput
+      Sampler and is recommended for most throughput-based use cases, because it
+      like the EMADynamicSampler, it maintains an Exponential Moving Average of
+      counts seen per key, and adjusts this average at regular intervals. The
+      weight applied to more recent intervals is defined by `weight`, a number
+      between (0, 1) - larger values weight the average more toward recent
+      observations. In other words, a larger weight will cause sample rates more
+      quickly adapt to traffic patterns, while a smaller weight will result in
+      sample rates that are less sensitive to bursts or drops in traffic and
+      thus more consistent over time.
+
+      New keys that are not already present in the EMA will always have a sample
+      rate of 1. Keys that occur more frequently will be sampled on a
+      logarithmic curve. Every key will be represented at least once in any
+      given window and more frequent keys will have their sample rate increased
+      proportionally to trend towards the goal throughput.
+    fields:
+      - name: GoalThroughputPerSec
+        type: int
+        validations:
+          - type: requiredInGroup
+        summary: is the desired throughput per second of events sent to Honeycomb.
+        description: >
+          The desired throughput **per second**. This is the number of events per
+          second you want to send to Honeycomb. The sampler will adjust sample
+          rates to try to achieve this desired throughput. This value is
+          calculated for the individual instance, not for the cluster; if your
+          cluster has multiple instances, you will need to divide your total
+          desired sample rate by the number of instances to get this value.
+      - name: InitialSampleRate
+        type: int
+        summary: is the sample rate to use.
+        description: >
+          InitialSampleRate is the sample rate to use during startup, before the
+          sampler has accumulated enough data to calculate a reasonable
+          throughput. This is mainly useful in situations where unsampled
+          throughput is high enough to cause problems.
+      - name: AdjustmentInterval
+        type: duration
+        summary: is how often the sampler will recalculate the sample rate.
+        description: >
+          The duration after which the EMAThroughput sampler should recalculate
+          its internal counters. It should be specified as a duration string,
+          e.g. "30s" or "1m".
+      - name: Weight
+        type: float
+        validations:
+          - type: minimum
+            arg: 0
+          - type: maximum
+            arg: 1
+        summary: is the weight to use when calculating the EMA.
+        description: >
+          The weight to use when calculating the EMA. It should be a number
+          between 0 and 1. Larger values weight the average more toward recent
+          observations. A larger weight will cause sample rates more quickly
+          adapt to traffic patterns, while a smaller weight will result in
+          sample rates that are less sensitive to bursts or drops in traffic and
+          thus more consistent over time.
+      - name: AgeOutValue
+        type: float
+        validations:
+          - type: minimum
+            arg: 0
+          - type: maximum
+            arg: 1
+        summary: is the value to use when aging out keys.
+        description: >
+          AgeOutValue indicates the threshold for removing keys from the EMA.
+          The EMA of any key will approach 0 if it is not repeatedly observed,
+          but will never truly reach it, so we have to decide what constitutes
+          "zero". Keys with averages below this threshold will be removed from
+          the EMA. Default is the same as Weight, as this prevents a key with
+          the smallest integer value (1) from being aged out immediately. This
+          value should generally be <= Weight, unless you have very specific
+          reasons to set it higher.
+      - name: BurstMultiple
+        type: float
+        summary: is the multiple of the sample rate to use when detecting bursts.
+        description: >
+          BurstMultiple, if set, is multiplied by the sum of the running average
+          of counts to define the burst detection threshold. If total counts
+          observed for a given interval exceed this threshold, EMA is updated
+          immediately, rather than waiting on the AdjustmentInterval. Defaults
+          to 2; a negative value disables. With the default of 2, if your traffic
+          suddenly doubles, burst detection will kick in.
+      - name: BurstDetectionDelay
+        type: int
+        summary: the number of AdjustmentIntervals to wait before detecting bursts.
+        description: >
+          Indicates the number of intervals to run before burst detection kicks
+          in. Defaults to 3.
+      - name: FieldList
+        type: stringarray
+        validations:
+          - type: requiredInGroup
+          - type: notempty
+        summary: is the list of fields to use to create the key for the dynamic sampler.
+        description: >
+          FieldList is a list of all the field names to use to form the key that
+          will be handed to the dynamic sampler. The combination of values from
+          all of these fields should reflect how interesting the trace is
+          compared to another. A good field selection has consistent values for
+          high-frequency, boring traffic, and unique values for outliers and
+          interesting traffic. Including an error field (or something like HTTP
+          status code) is an excellent choice. Using fields with very high
+          cardinality (like `k8s.pod.id`), is a bad choice. If the combination
+          of fields essentially makes them unique, the dynamic sampler will
+          sample everything. If the combination of fields is not unique enough,
+          you will not be guaranteed samples of the most interesting traces. As
+          an example, consider a combination of HTTP endpoint (high-frequency
+          and boring), HTTP method, and status code (normally boring but can
+          become interesting when indicating an error) as a good set of fields
+          since it will allowing proper sampling of all endpoints under normal
+          traffic and call out when there is failing traffic to any endpoint.
+          For example, in contrast, consider a combination of HTTP endpoint,
+          status code, and pod id as a bad set of fields, since it would result
+          in keys that are all unique, and therefore results in sampling 100% of
+          traces. Using only the HTTP endpoint field would be a **bad** choice,
+          as it is not unique enough and therefore interesting traces, like
+          traces that experienced a `500`, might not be sampled. Field names may
+          come from any span in the trace; if they occur on multiple spans, all
+          unique values will be included in the key.
+      - name: MaxKeys
+        type: int
+        summary: is the maximum number of keys to track.
+        description: >
+          MaxKeys limits the number of distinct keys tracked by the sampler.
+          Once MaxKeys is reached, new keys will not be included in the sample
+          rate map, but existing keys will continue to be be counted. You can
+          use this to keep the sample rate map size under control. Defaults to
+          500; dynamic samplers will rarely achieve their goals with more keys
+          than this.
+      - name: UseTraceLength
+        type: bool
+        summary: indicates whether to include the trace length as part of the key.
+        description: >
+          Indicates whether to include the trace length (number of spans in the
+          trace) as part of the key. The number of spans is exact, so if there
+          are normally small variations in trace length you may want to leave
+          this off. If traces are consistent lengths and changes in trace length
+          is a useful indicator of traces you'd like to see in Honeycomb, set
+          this to true.
 
   - name: TotalThroughputSampler
     title: Total Throughput Sampler
@@ -317,11 +471,12 @@ groups:
         type: int
         summary: is the maximum number of keys to track.
         description: >
-          MaxKeys, if greater than 0, limits the number of distinct keys tracked
-          by the sampler. Once MaxKeys is reached, new keys will not be included
-          in the sample rate map, but existing keys will continue to be be
-          counted. You can use this to keep the sample rate map size under
-          control.
+          MaxKeys limits the number of distinct keys tracked by the sampler.
+          Once MaxKeys is reached, new keys will not be included in the sample
+          rate map, but existing keys will continue to be be counted. You can
+          use this to keep the sample rate map size under control. Defaults to
+          500; dynamic samplers will rarely achieve their goals with more keys
+          than this.
       - name: UseTraceLength
         type: bool
         summary: indicates whether to include the trace length as part of the key.
diff --git a/config/sampler_config.go b/config/sampler_config.go
index 0f9be4d..9116092 100644
--- a/config/sampler_config.go
+++ b/config/sampler_config.go
@@ -15,6 +15,7 @@ type V2SamplerChoice struct {
 	RulesBasedSampler      *RulesBasedSamplerConfig      `json:"rulesbasedsampler" yaml:"RulesBasedSampler,omitempty"`
 	DynamicSampler         *DynamicSamplerConfig         `json:"dynamicsampler" yaml:"DynamicSampler,omitempty"`
 	EMADynamicSampler      *EMADynamicSamplerConfig      `json:"emadynamicsampler" yaml:"EMADynamicSampler,omitempty"`
+	EMAThroughputSampler   *EMAThroughputSamplerConfig   `json:"emathroughputsampler" yaml:"EMAThroughputSampler,omitempty"`
 	TotalThroughputSampler *TotalThroughputSamplerConfig `json:"totalthroughputsampler" yaml:"TotalThroughputSampler,omitempty"`
 }
 
@@ -28,6 +29,8 @@ func (v *V2SamplerChoice) Sampler() (any, string) {
 		return v.DynamicSampler, "DynamicSampler"
 	case v.EMADynamicSampler != nil:
 		return v.EMADynamicSampler, "EMADynamicSampler"
+	case v.EMAThroughputSampler != nil:
+		return v.EMAThroughputSampler, "EMAThroughputSampler"
 	case v.TotalThroughputSampler != nil:
 		return v.TotalThroughputSampler, "TotalThroughputSampler"
 	default:
@@ -64,6 +67,19 @@ type EMADynamicSamplerConfig struct {
 	UseTraceLength      bool     `json:"usetracelength" yaml:"UseTraceLength,omitempty"`
 }
 
+type EMAThroughputSamplerConfig struct {
+	GoalThroughputPerSec int      `json:"goalthroughputpersec" yaml:"GoalThroughputPerSec,omitempty"`
+	InitialSampleRate    int      `json:"initialsamplerate" yaml:"InitialSampleRate,omitempty"`
+	AdjustmentInterval   Duration `json:"adjustmentinterval" yaml:"AdjustmentInterval,omitempty"`
+	Weight               float64  `json:"weight" yaml:"Weight,omitempty"`
+	AgeOutValue          float64  `json:"ageoutvalue" yaml:"AgeOutValue,omitempty"`
+	BurstMultiple        float64  `json:"burstmultiple" yaml:"BurstMultiple,omitempty"`
+	BurstDetectionDelay  uint     `json:"burstdetectiondelay" yaml:"BurstDetectionDelay,omitempty"`
+	FieldList            []string `json:"fieldlist" yaml:"FieldList,omitempty"`
+	MaxKeys              int      `json:"maxkeys" yaml:"MaxKeys,omitempty"`
+	UseTraceLength       bool     `json:"usetracelength" yaml:"UseTraceLength,omitempty"`
+}
+
 type TotalThroughputSamplerConfig struct {
 	GoalThroughputPerSec int64    `json:"goalthroughputpersec" yaml:"GoalThroughputPerSec,omitempty" validate:"gte=1"`
 	ClearFrequency       Duration `json:"clearfrequency" yaml:"ClearFrequency,omitempty"`
@@ -81,6 +97,7 @@ type RulesBasedSamplerConfig struct {
 type RulesBasedDownstreamSampler struct {
 	DynamicSampler         *DynamicSamplerConfig         `json:"dynamicsampler" yaml:"DynamicSampler,omitempty"`
 	EMADynamicSampler      *EMADynamicSamplerConfig      `json:"emadynamicsampler" yaml:"EMADynamicSampler,omitempty"`
+	EMAThroughputSampler   *EMAThroughputSamplerConfig   `json:"emathroughputsampler" yaml:"EMAThroughputSampler,omitempty"`
 	TotalThroughputSampler *TotalThroughputSamplerConfig `json:"totalthroughputsampler" yaml:"TotalThroughputSampler,omitempty"`
 }
 
diff --git a/go.mod b/go.mod
index d89137b..b266a3e 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/gomodule/redigo v1.8.9
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/honeycombio/dynsampler-go v0.4.0
+	github.com/honeycombio/dynsampler-go v0.5.0
 	github.com/honeycombio/husky v0.22.4
 	github.com/honeycombio/libhoney-go v1.18.0
 	github.com/jessevdk/go-flags v1.5.0
diff --git a/go.sum b/go.sum
index f83659f..643f946 100644
--- a/go.sum
+++ b/go.sum
@@ -75,6 +75,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/honeycombio/dynsampler-go v0.4.0 h1:80Hj9M9r23deTUXV7A22pY07agtCxKBjXn/SeC4w5zc=
 github.com/honeycombio/dynsampler-go v0.4.0/go.mod h1:8AoR25Xt1YR6LcCoPIm2rkozXXjSPCb/xFp5g8BSsXo=
+github.com/honeycombio/dynsampler-go v0.5.0 h1:wBeUz0PPdp6FayDuO8X5VJLCNJqTE4KA4gCbNNkWi1Y=
+github.com/honeycombio/dynsampler-go v0.5.0/go.mod h1:pJqWFeoMN3syX74PEvlusieyGBbtIBjmTVjLc3thmK4=
 github.com/honeycombio/husky v0.22.4 h1:2ZEenP3y1KD8xrP2nGvjeB4K/6F3Zfhgu6an2e7cSsI=
 github.com/honeycombio/husky v0.22.4/go.mod h1:oIb9sbjnC3SdlmLd84JHF7MCaitRsvTnrNyO+yxWO0M=
 github.com/honeycombio/libhoney-go v1.18.0 h1:OYHOP381r3Ea76BhUYeza8PUTMDp8MByoOxDn3qtEq8=
diff --git a/rules_complete.yaml b/rules_complete.yaml
index 3e3de4c..9fbd5cf 100644
--- a/rules_complete.yaml
+++ b/rules_complete.yaml
@@ -106,7 +106,9 @@ Samplers:
                 - Name: default rule
                   SampleRate: 10
     env5:
-        TotalThroughputSampler:
+        # This shows using the EMAThroughput sampler to set a goal throughput,
+        # using mostly default values.
+        EMAThroughputSampler:
             GoalThroughputPerSec: 100
             FieldList:
                 - request.method
diff --git a/sample/ema_throughput.go b/sample/ema_throughput.go
new file mode 100644
index 0000000..bf7acce
--- /dev/null
+++ b/sample/ema_throughput.go
@@ -0,0 +1,88 @@
+package sample
+
+import (
+	"math/rand"
+	"time"
+
+	dynsampler "github.com/honeycombio/dynsampler-go"
+
+	"github.com/honeycombio/refinery/config"
+	"github.com/honeycombio/refinery/logger"
+	"github.com/honeycombio/refinery/metrics"
+	"github.com/honeycombio/refinery/types"
+)
+
+type EMAThroughputSampler struct {
+	Config  *config.EMAThroughputSamplerConfig
+	Logger  logger.Logger
+	Metrics metrics.Metrics
+
+	adjustmentInterval   config.Duration
+	weight               float64
+	initialSampleRate    int
+	goalthroughputpersec int
+	ageOutValue          float64
+	burstMultiple        float64
+	burstDetectionDelay  uint
+	maxKeys              int
+
+	key *traceKey
+
+	dynsampler dynsampler.Sampler
+}
+
+func (d *EMAThroughputSampler) Start() error {
+	d.Logger.Debug().Logf("Starting EMAThroughputSampler")
+	defer func() { d.Logger.Debug().Logf("Finished starting EMAThroughputSampler") }()
+	d.initialSampleRate = d.Config.InitialSampleRate
+	d.goalthroughputpersec = d.Config.GoalThroughputPerSec
+	d.adjustmentInterval = d.Config.AdjustmentInterval
+	d.weight = d.Config.Weight
+	d.ageOutValue = d.Config.AgeOutValue
+	d.burstMultiple = d.Config.BurstMultiple
+	d.burstDetectionDelay = d.Config.BurstDetectionDelay
+	d.maxKeys = d.Config.MaxKeys
+	d.key = newTraceKey(d.Config.FieldList, d.Config.UseTraceLength)
+
+	// spin up the actual dynamic sampler
+	d.dynsampler = &dynsampler.EMAThroughput{
+		GoalThroughputPerSec: d.goalthroughputpersec,
+		InitialSampleRate:    d.initialSampleRate,
+		AdjustmentInterval:   time.Duration(d.adjustmentInterval),
+		Weight:               d.weight,
+		AgeOutValue:          d.ageOutValue,
+		BurstDetectionDelay:  d.burstDetectionDelay,
+		BurstMultiple:        d.burstMultiple,
+		MaxKeys:              d.maxKeys,
+	}
+	d.dynsampler.Start()
+
+	// Register statistics this package will produce
+	d.Metrics.Register("dynsampler_num_dropped", "counter")
+	d.Metrics.Register("dynsampler_num_kept", "counter")
+	d.Metrics.Register("dynsampler_sample_rate", "histogram")
+
+	return nil
+}
+
+func (d *EMAThroughputSampler) GetSampleRate(trace *types.Trace) (rate uint, keep bool, reason string, key string) {
+	key = d.key.build(trace)
+	rate = uint(d.dynsampler.GetSampleRate(key))
+	if rate < 1 { // protect against dynsampler being broken even though it shouldn't be
+		rate = 1
+	}
+	shouldKeep := rand.Intn(int(rate)) == 0
+	d.Logger.Debug().WithFields(map[string]interface{}{
+		"sample_key":  key,
+		"sample_rate": rate,
+		"sample_keep": shouldKeep,
+		"trace_id":    trace.TraceID,
+	}).Logf("got sample rate and decision")
+	if shouldKeep {
+		d.Metrics.Increment("dynsampler_num_kept")
+	} else {
+		d.Metrics.Increment("dynsampler_num_dropped")
+	}
+	d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
+	return rate, shouldKeep, "emathroughput", key
+}
diff --git a/tools/convert/rules.md b/tools/convert/rules.md
index 1a6c530..5d7cc03 100644
--- a/tools/convert/rules.md
+++ b/tools/convert/rules.md
@@ -122,11 +122,12 @@ Type: `stringarray`
 
 MaxKeys is the maximum number of keys to track.
 
-MaxKeys, if greater than 0, limits the number of distinct keys tracked
-by the sampler. Once MaxKeys is reached, new keys will not be included
-in the sample rate map, but existing keys will continue to be be
-counted. You can use this to keep the sample rate map size under
-control.
+MaxKeys limits the number of distinct keys tracked by the sampler.
+Once MaxKeys is reached, new keys will not be included in the sample
+rate map, but existing keys will continue to be be counted. You can
+use this to keep the sample rate map size under control. Defaults to
+500; dynamic samplers will rarely achieve their goals with more keys
+than this.
 
 
 Not eligible for live reload.
@@ -175,10 +176,9 @@ will result in sample rates that are less sensitive to bursts or drops
 in traffic and thus more consistent over time.
 Keys that are not already present in the EMA will always have a sample
 rate of 1. Keys that occur more frequently will be sampled on a
-logarithmic curve. In other words, every key will be represented at
-least once in any given window and more frequent keys will have their
-sample rate increased proportionally to trend towards the goal sample
-rate.
+logarithmic curve. Every key will be represented at least once in any
+given window and more frequent keys will have their sample rate
+increased proportionally to trend towards the goal sample rate.
 
 
 ### GoalSampleRate
@@ -335,11 +335,12 @@ Type: `stringarray`
 
 MaxKeys is the maximum number of keys to track.
 
-MaxKeys, if greater than 0, limits the number of distinct keys tracked
-by the sampler. Once MaxKeys is reached, new keys will not be included
-in the sample rate map, but existing keys will continue to be be
-counted. You can use this to keep the sample rate map size under
-control.
+MaxKeys limits the number of distinct keys tracked by the sampler.
+Once MaxKeys is reached, new keys will not be included in the sample
+rate map, but existing keys will continue to be be counted. You can
+use this to keep the sample rate map size under control. Defaults to
+500; dynamic samplers will rarely achieve their goals with more keys
+than this.
 
 
 Not eligible for live reload.
@@ -359,7 +360,239 @@ trace) as part of the key. The number of spans is exact, so if there
 are normally small variations in trace length you may want to leave
 this off. If traces are consistent lengths and changes in trace length
 is a useful indicator of traces you'd like to see in Honeycomb, set
-this to true. Eligible for live reload.
+this to true.
+
+
+Not eligible for live reload.
+
+Type: `bool`
+
+
+
+
+---
+## EMAThroughputSampler: EMA Throughput Sampler
+
+The Exponential Moving Average (EMA) Throughput Sampler attempts to
+achieve a given throughput -- number of spans per second -- weighting
+rare traffic and frequent traffic differently so as to end up with the
+correct rate.
+The EMAThroughputSampler is an improvement upon the TotalThroughput
+Sampler and is recommended for most throughput-based use cases,
+because it like the EMADynamicSampler, it maintains an Exponential
+Moving Average of counts seen per key, and adjusts this average at
+regular intervals. The weight applied to more recent intervals is
+defined by `weight`, a number between (0, 1) - larger values weight
+the average more toward recent observations. In other words, a larger
+weight will cause sample rates more quickly adapt to traffic patterns,
+while a smaller weight will result in sample rates that are less
+sensitive to bursts or drops in traffic and thus more consistent over
+time.
+New keys that are not already present in the EMA will always have a
+sample rate of 1. Keys that occur more frequently will be sampled on a
+logarithmic curve. Every key will be represented at least once in any
+given window and more frequent keys will have their sample rate
+increased proportionally to trend towards the goal throughput.
+
+
+### GoalThroughputPerSec
+
+GoalThroughputPerSec is the desired throughput per second of events
+sent to Honeycomb.
+
+The desired throughput **per second**. This is the number of events
+per second you want to send to Honeycomb. The sampler will adjust
+sample rates to try to achieve this desired throughput. This value is
+calculated for the individual instance, not for the cluster; if your
+cluster has multiple instances, you will need to divide your total
+desired sample rate by the number of instances to get this value.
+
+
+Not eligible for live reload.
+
+Type: `int`
+
+
+
+
+### InitialSampleRate
+
+InitialSampleRate is the sample rate to use.
+
+InitialSampleRate is the sample rate to use during startup, before the
+sampler has accumulated enough data to calculate a reasonable
+throughput. This is mainly useful in situations where unsampled
+throughput is high enough to cause problems.
+
+
+Not eligible for live reload.
+
+Type: `int`
+
+
+
+
+### AdjustmentInterval
+
+AdjustmentInterval is how often the sampler will recalculate the
+sample rate.
+
+The duration after which the EMAThroughput sampler should recalculate
+its internal counters. It should be specified as a duration string,
+e.g. "30s" or "1m".
+
+
+Not eligible for live reload.
+
+Type: `duration`
+
+
+
+
+### Weight
+
+Weight is the weight to use when calculating the EMA.
+
+The weight to use when calculating the EMA. It should be a number
+between 0 and 1. Larger values weight the average more toward recent
+observations. A larger weight will cause sample rates more quickly
+adapt to traffic patterns, while a smaller weight will result in
+sample rates that are less sensitive to bursts or drops in traffic and
+thus more consistent over time.
+
+
+Not eligible for live reload.
+
+Type: `float`
+
+
+
+
+### AgeOutValue
+
+AgeOutValue is the value to use when aging out keys.
+
+AgeOutValue indicates the threshold for removing keys from the EMA.
+The EMA of any key will approach 0 if it is not repeatedly observed,
+but will never truly reach it, so we have to decide what constitutes
+"zero". Keys with averages below this threshold will be removed from
+the EMA. Default is the same as Weight, as this prevents a key with
+the smallest integer value (1) from being aged out immediately. This
+value should generally be <= Weight, unless you have very specific
+reasons to set it higher.
+
+
+Not eligible for live reload.
+
+Type: `float`
+
+
+
+
+### BurstMultiple
+
+BurstMultiple is the multiple of the sample rate to use when detecting
+bursts.
+
+BurstMultiple, if set, is multiplied by the sum of the running average
+of counts to define the burst detection threshold. If total counts
+observed for a given interval exceed this threshold, EMA is updated
+immediately, rather than waiting on the AdjustmentInterval. Defaults
+to 2; a negative value disables. With the default of 2, if your
+traffic suddenly doubles, burst detection will kick in.
+
+
+Not eligible for live reload.
+
+Type: `float`
+
+
+
+
+### BurstDetectionDelay
+
+BurstDetectionDelay the number of AdjustmentIntervals to wait before
+detecting bursts.
+
+Indicates the number of intervals to run before burst detection kicks
+in. Defaults to 3.
+
+
+Not eligible for live reload.
+
+Type: `int`
+
+
+
+
+### FieldList
+
+FieldList is the list of fields to use to create the key for the
+dynamic sampler.
+
+FieldList is a list of all the field names to use to form the key that
+will be handed to the dynamic sampler. The combination of values from
+all of these fields should reflect how interesting the trace is
+compared to another. A good field selection has consistent values for
+high-frequency, boring traffic, and unique values for outliers and
+interesting traffic. Including an error field (or something like HTTP
+status code) is an excellent choice. Using fields with very high
+cardinality (like `k8s.pod.id`), is a bad choice. If the combination
+of fields essentially makes them unique, the dynamic sampler will
+sample everything. If the combination of fields is not unique enough,
+you will not be guaranteed samples of the most interesting traces. As
+an example, consider a combination of HTTP endpoint (high-frequency
+and boring), HTTP method, and status code (normally boring but can
+become interesting when indicating an error) as a good set of fields
+since it will allowing proper sampling of all endpoints under normal
+traffic and call out when there is failing traffic to any endpoint.
+For example, in contrast, consider a combination of HTTP endpoint,
+status code, and pod id as a bad set of fields, since it would result
+in keys that are all unique, and therefore results in sampling 100% of
+traces. Using only the HTTP endpoint field would be a **bad** choice,
+as it is not unique enough and therefore interesting traces, like
+traces that experienced a `500`, might not be sampled. Field names may
+come from any span in the trace; if they occur on multiple spans, all
+unique values will be included in the key.
+
+
+Not eligible for live reload.
+
+Type: `stringarray`
+
+
+
+
+### MaxKeys
+
+MaxKeys is the maximum number of keys to track.
+
+MaxKeys limits the number of distinct keys tracked by the sampler.
+Once MaxKeys is reached, new keys will not be included in the sample
+rate map, but existing keys will continue to be be counted. You can
+use this to keep the sample rate map size under control. Defaults to
+500; dynamic samplers will rarely achieve their goals with more keys
+than this.
+
+
+Not eligible for live reload.
+
+Type: `int`
+
+
+
+
+### UseTraceLength
+
+UseTraceLength indicates whether to include the trace length as part
+of the key.
+
+Indicates whether to include the trace length (number of spans in the
+trace) as part of the key. The number of spans is exact, so if there
+are normally small variations in trace length you may want to leave
+this off. If traces are consistent lengths and changes in trace length
+is a useful indicator of traces you'd like to see in Honeycomb, set
+this to true.
 
 
 Not eligible for live reload.
@@ -463,11 +696,12 @@ Type: `stringarray`
 
 MaxKeys is the maximum number of keys to track.
 
-MaxKeys, if greater than 0, limits the number of distinct keys tracked
-by the sampler. Once MaxKeys is reached, new keys will not be included
-in the sample rate map, but existing keys will continue to be be
-counted. You can use this to keep the sample rate map size under
-control.
+MaxKeys limits the number of distinct keys tracked by the sampler.
+Once MaxKeys is reached, new keys will not be included in the sample
+rate map, but existing keys will continue to be be counted. You can
+use this to keep the sample rate map size under control. Defaults to
+500; dynamic samplers will rarely achieve their goals with more keys
+than this.
 
 
 Not eligible for live reload.
@@ -733,7 +967,7 @@ Type: `string`
 
 
 ---
-## Samplers:
+## Samplers: 
 
 Samplers is a mapping of targets to samplers. Each target is a
 Honeycomb environment (or, for classic keys, a dataset). The value is


Test patch:
diff --git a/route/route_test.go b/route/route_test.go
index c5d5ce1..85d11d6 100644
--- a/route/route_test.go
+++ b/route/route_test.go
@@ -328,7 +328,7 @@ func TestDebugAllRules(t *testing.T) {
 	}{
 		{
 			format: "json",
-			expect: `{"rulesversion":0,"samplers":{"dataset1":{"deterministicsampler":{"samplerate":0},"rulesbasedsampler":null,"dynamicsampler":null,"emadynamicsampler":null,"totalthroughputsampler":null}}}`,
+			expect: `{"rulesversion":0,"samplers":{"dataset1":{"deterministicsampler":{"samplerate":0},"rulesbasedsampler":null,"dynamicsampler":null,"emadynamicsampler":null,"emathroughputsampler":null,"totalthroughputsampler":null}}}`,
 		},
 		{
 			format: "toml",
diff --git a/sample/ema_throughput_test.go b/sample/ema_throughput_test.go
new file mode 100644
index 0000000..152e439
--- /dev/null
+++ b/sample/ema_throughput_test.go
@@ -0,0 +1,51 @@
+package sample
+
+import (
+	"testing"
+
+	"github.com/honeycombio/refinery/config"
+	"github.com/honeycombio/refinery/logger"
+	"github.com/honeycombio/refinery/metrics"
+	"github.com/honeycombio/refinery/types"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEMAThroughputAddSampleRateKeyToTrace(t *testing.T) {
+	const spanCount = 5
+
+	metrics := metrics.MockMetrics{}
+	metrics.Start()
+
+	sampler := &EMAThroughputSampler{
+		Config: &config.EMAThroughputSamplerConfig{
+			FieldList: []string{"http.status_code", "request.path", "app.team.id", "important_field"},
+		},
+		Logger:  &logger.NullLogger{},
+		Metrics: &metrics,
+	}
+
+	trace := &types.Trace{}
+	for i := 0; i < spanCount; i++ {
+		trace.AddSpan(&types.Span{
+			Event: types.Event{
+				Data: map[string]interface{}{
+					"http.status_code": 200,
+					"app.team.id":      float64(4),
+					"important_field":  true,
+					"request.path":     "/{slug}/fun",
+				},
+			},
+		})
+	}
+	sampler.Start()
+	rate, _, reason, key := sampler.GetSampleRate(trace)
+
+	spans := trace.GetSpans()
+
+	assert.Len(t, spans, spanCount, "should have the same number of spans as input")
+	assert.Equal(t, uint(10), rate, "sample rate should be 10")
+	assert.Equal(t, "emathroughput", reason)
+	assert.Equal(t, "4•,200•,true•,/{slug}/fun•,", key)
+
+}