Repository: kubernetes-sigs/node-feature-discovery
Pull request #663 (2021-11-22 12:55:41)

Add variables to feature rule spec and support backrefs 
Support backreferencing of output values from previous rules. Enables
complex rule setups where custom features are further combined together
to form even more sophisticated higher level labels. The labels created
by preceding rules are available as a special 'rule.matched' feature
(for matchFeatures to use).

If referencing rules accross multiple configs/CRDs care must be taken
with the ordering. Processing order of rules in nfd-worker:

1. Static rules
2. Files from /etc/kubernetes/node-feature-discovery/custom.d/
   in alphabetical order. Subdirectories are processed by reading their
   files in alphabetical order.
3. Custom rules from main nfd-worker.conf

In nfd-master, NodeFeatureRule objects are processed in alphabetical
order (based on their metadata.name).

This patch also adds new 'vars' fields to the rule spec. Like 'labels',
it is a map of key-value pairs but no labels are generated from these.
The values specified in 'vars' are only added for backreferencing into
the 'rules.matched' feature. This may by desired in schemes where the
output of certain rules is only used as intermediate variables for other
rules and no labels out of these are wanted.

An example setup:

```
  - name: "kernel feature"
    labels:
      kernel-feature:
    matchFeatures:
      - feature: kernel.version
        matchExpressions:
          major: {op: Gt, value: ["4"]}

  - name: "intermediate var feature"
    vars:
      nolabel-feature: "true"
    matchFeatures:
      - feature: cpu.cpuid
        matchExpressions:
          AVX512F: {op: Exists}
      - feature: pci.device
        matchExpressions:
          vendor: {op: In, value: ["8086"]}
          device: {op: In, value: ["1234", "1235"]}

  - name: top-level-feature
    matchFeatures:
      - feature: rule.matched
        matchExpressions:
          kernel-feature: "true"
          nolabel-feature: "true"
```

k8s-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/node-feature-discovery/pull/663#" title="Author self-approved">marquiz</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fnode-feature-discovery).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/node-feature-discovery/blob/master/OWNERS)~~ [marquiz]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
marquiz: Depends on #553 
/hold
k8s-ci-robot: @marquiz: PR needs rebase.

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository.
</details>
marquiz: /unhold
ArangoGutierrez: @marquiz rebase 😁 
/assign
k8s-ci-robot: @marquiz: PR needs rebase.

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository.
</details>
marquiz: > @marquiz rebase 😁

Rebased. Also added support for `varsTemplate`.

Need to double-verify this and add unit tests
/hold
marquiz: Basic unit tests added.

This depends on #668 so still on hold

Patch:
diff --git a/deployment/base/nfd-crds/cr-sample.yaml b/deployment/base/nfd-crds/cr-sample.yaml
index c6050d000..dad9405d0 100644
--- a/deployment/base/nfd-crds/cr-sample.yaml
+++ b/deployment/base/nfd-crds/cr-sample.yaml
@@ -111,3 +111,29 @@ spec:
         - feature: cpu.cpuid
           matchExpressions:
             AVX: {op: Exists}
+
+    # The following examples demonstrate vars field and back-referencing
+    # previous labels and vars
+    - name: "my dummy kernel rule"
+      labels:
+        "my.kernel.feature": "true"
+      matchFeatures:
+        - feature: kernel.version
+          matchExpressions:
+            major: {op: Gt, value: ["2"]}
+
+    - name: "my dummy rule with no labels"
+      vars:
+        "my.dummy.var": "1"
+      matchFeatures:
+        - feature: cpu.cpuid
+          matchExpressions: {}
+
+    - name: "my rule using backrefs"
+      labels:
+        "my.backref.feature": "true"
+      matchFeatures:
+        - feature: rule.matched
+          matchExpressions:
+            my.kernel.feature: {op: IsTrue}
+            my.dummy.var: {op: Gt, value: ["0"]}
diff --git a/deployment/base/nfd-crds/nodefeaturerule-crd.yaml b/deployment/base/nfd-crds/nodefeaturerule-crd.yaml
index 277a1508f..cd9b48bf4 100644
--- a/deployment/base/nfd-crds/nodefeaturerule-crd.yaml
+++ b/deployment/base/nfd-crds/nodefeaturerule-crd.yaml
@@ -188,6 +188,21 @@ spec:
                     name:
                       description: Name of the rule.
                       type: string
+                    vars:
+                      additionalProperties:
+                        type: string
+                      description: Vars is the variables to store if the rule matches.
+                        Variables do not directly inflict any changes in the node
+                        object. However, they can be referenced from other rules enabling
+                        more complex rule hierarchies, without exposing intermediary
+                        output values as labels.
+                      type: object
+                    varsTemplate:
+                      description: VarsTemplate specifies a template to expand for
+                        dynamically generating multiple variables. Data (after template
+                        expansion) must be keys with an optional value (<key>[=<value>])
+                        separated by newlines.
+                      type: string
                   required:
                   - name
                   type: object
diff --git a/deployment/components/worker-config/nfd-worker.conf.example b/deployment/components/worker-config/nfd-worker.conf.example
index 9586163e9..2a5d2961b 100644
--- a/deployment/components/worker-config/nfd-worker.conf.example
+++ b/deployment/components/worker-config/nfd-worker.conf.example
@@ -226,3 +226,29 @@
 #        - feature: cpu.cpuid
 #          matchExpressions:
 #            AVX: {op: Exists}
+#
+#    # The following examples demonstrate vars field and back-referencing
+#    # previous labels and vars
+#    - name: "my dummy kernel rule"
+#      labels:
+#        "my.kernel.feature": "true"
+#      matchFeatures:
+#        - feature: kernel.version
+#          matchExpressions:
+#            major: {op: Gt, value: ["2"]}
+#
+#    - name: "my dummy rule with no labels"
+#      vars:
+#        "my.dummy.var": "1"
+#      matchFeatures:
+#        - feature: cpu.cpuid
+#          matchExpressions: {}
+#
+#    - name: "my rule using backrefs"
+#      labels:
+#        "my.backref.feature": "true"
+#      matchFeatures:
+#        - feature: rule.matched
+#          matchExpressions:
+#            my.kernel.feature: {op: IsTrue}
+#            my.dummy.var: {op: Gt, value: ["0"]}
diff --git a/deployment/helm/node-feature-discovery/manifests/nodefeaturerule-crd.yaml b/deployment/helm/node-feature-discovery/manifests/nodefeaturerule-crd.yaml
index 277a1508f..cd9b48bf4 100644
--- a/deployment/helm/node-feature-discovery/manifests/nodefeaturerule-crd.yaml
+++ b/deployment/helm/node-feature-discovery/manifests/nodefeaturerule-crd.yaml
@@ -188,6 +188,21 @@ spec:
                     name:
                       description: Name of the rule.
                       type: string
+                    vars:
+                      additionalProperties:
+                        type: string
+                      description: Vars is the variables to store if the rule matches.
+                        Variables do not directly inflict any changes in the node
+                        object. However, they can be referenced from other rules enabling
+                        more complex rule hierarchies, without exposing intermediary
+                        output values as labels.
+                      type: object
+                    varsTemplate:
+                      description: VarsTemplate specifies a template to expand for
+                        dynamically generating multiple variables. Data (after template
+                        expansion) must be keys with an optional value (<key>[=<value>])
+                        separated by newlines.
+                      type: string
                   required:
                   - name
                   type: object
diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml
index 3afce9106..b7b50142b 100644
--- a/deployment/helm/node-feature-discovery/values.yaml
+++ b/deployment/helm/node-feature-discovery/values.yaml
@@ -315,6 +315,32 @@ worker:
     #        - feature: cpu.cpuid
     #          matchExpressions:
     #            AVX: {op: Exists}
+    #
+    #    # The following examples demonstrate vars field and back-referencing
+    #    # previous labels and vars
+    #    - name: "my dummy kernel rule"
+    #      labels:
+    #        "my.kernel.feature": "true"
+    #      matchFeatures:
+    #        - feature: kernel.version
+    #          matchExpressions:
+    #            major: {op: Gt, value: ["2"]}
+    #
+    #    - name: "my dummy rule with no labels"
+    #      vars:
+    #        "my.dummy.var": "1"
+    #      matchFeatures:
+    #        - feature: cpu.cpuid
+    #          matchExpressions: {}
+    #
+    #    - name: "my rule using backrefs"
+    #      labels:
+    #        "my.backref.feature": "true"
+    #      matchFeatures:
+    #        - feature: rule.matched
+    #          matchExpressions:
+    #            my.kernel.feature: {op: IsTrue}
+    #            my.dummy.var: {op: Gt, value: ["0"]}
 ### <NFD-WORKER-CONF-END-DO-NOT-REMOVE>
 
   podSecurityContext: {}
diff --git a/pkg/api/feature/feature.go b/pkg/api/feature/feature.go
index d90339e22..72340ee58 100644
--- a/pkg/api/feature/feature.go
+++ b/pkg/api/feature/feature.go
@@ -50,3 +50,18 @@ func NewInstanceFeature(attrs map[string]string) *InstanceFeature {
 	}
 	return &InstanceFeature{Attributes: attrs}
 }
+
+// InsertFeatureValues inserts new values into a specific feature.
+func InsertFeatureValues(f Features, domain, feature string, values map[string]string) {
+	if _, ok := f[domain]; !ok {
+		f[domain] = NewDomainFeatures()
+	}
+	if _, ok := f[domain].Values[feature]; !ok {
+		f[domain].Values[feature] = NewValueFeatures(values)
+		return
+	}
+
+	for k, v := range values {
+		f[domain].Values[feature].Elements[k] = v
+	}
+}
diff --git a/pkg/apis/nfd/v1alpha1/rule.go b/pkg/apis/nfd/v1alpha1/rule.go
index 704b3d01d..0f6f81cd7 100644
--- a/pkg/apis/nfd/v1alpha1/rule.go
+++ b/pkg/apis/nfd/v1alpha1/rule.go
@@ -17,26 +17,35 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"bytes"
+	"fmt"
 	"strings"
 	"text/template"
 
-	"bytes"
-	"fmt"
+	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/node-feature-discovery/pkg/api/feature"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
 )
 
+// RuleOutput contains the output out rule execution.
+// +k8s:deepcopy-gen=false
+type RuleOutput struct {
+	Labels map[string]string
+	Vars   map[string]string
+}
+
 // Execute the rule against a set of input features.
-func (r *Rule) Execute(features map[string]*feature.DomainFeatures) (map[string]string, error) {
-	ret := make(map[string]string)
+func (r *Rule) Execute(features feature.Features) (RuleOutput, error) {
+	labels := make(map[string]string)
+	vars := make(map[string]string)
 
 	if len(r.MatchAny) > 0 {
 		// Logical OR over the matchAny matchers
 		matched := false
 		for _, matcher := range r.MatchAny {
 			if m, err := matcher.match(features); err != nil {
-				return nil, err
+				return RuleOutput{}, err
 			} else if m != nil {
 				matched = true
 				utils.KlogDump(4, "matches for matchAny "+r.Name, "  ", m)
@@ -46,34 +55,47 @@ func (r *Rule) Execute(features map[string]*feature.DomainFeatures) (map[string]
 					// produce the same labels)
 					break
 				}
-				if err := r.executeLabelsTemplate(m, ret); err != nil {
-					return nil, err
+				if err := r.executeLabelsTemplate(m, labels); err != nil {
+					return RuleOutput{}, err
+				}
+				if err := r.executeVarsTemplate(m, vars); err != nil {
+					return RuleOutput{}, err
 				}
-
 			}
 		}
 		if !matched {
-			return nil, nil
+			klog.V(2).Infof("rule %q did not match", r.Name)
+			return RuleOutput{}, nil
 		}
 	}
 
 	if len(r.MatchFeatures) > 0 {
 		if m, err := r.MatchFeatures.match(features); err != nil {
-			return nil, err
+			return RuleOutput{}, err
 		} else if m == nil {
-			return nil, nil
+			klog.V(2).Infof("rule %q did not match", r.Name)
+			return RuleOutput{}, nil
 		} else {
 			utils.KlogDump(4, "matches for matchFeatures "+r.Name, "  ", m)
-			if err := r.executeLabelsTemplate(m, ret); err != nil {
-				return nil, err
+			if err := r.executeLabelsTemplate(m, labels); err != nil {
+				return RuleOutput{}, err
+			}
+			if err := r.executeVarsTemplate(m, vars); err != nil {
+				return RuleOutput{}, err
 			}
 		}
 	}
 
 	for k, v := range r.Labels {
-		ret[k] = v
+		labels[k] = v
+	}
+	for k, v := range r.Vars {
+		vars[k] = v
 	}
 
+	ret := RuleOutput{Labels: labels, Vars: vars}
+	utils.KlogDump(2, fmt.Sprintf("rule %q matched with: ", r.Name), "  ", ret)
+
 	return ret, nil
 }
 
@@ -100,6 +122,28 @@ func (r *Rule) executeLabelsTemplate(in matchedFeatures, out map[string]string)
 	return nil
 }
 
+func (r *Rule) executeVarsTemplate(in matchedFeatures, out map[string]string) error {
+	if r.VarsTemplate == "" {
+		return nil
+	}
+	if r.varsTemplate == nil {
+		t, err := newTemplateHelper(r.VarsTemplate)
+		if err != nil {
+			return err
+		}
+		r.varsTemplate = t
+	}
+
+	vars, err := r.varsTemplate.expandMap(in)
+	if err != nil {
+		return err
+	}
+	for k, v := range vars {
+		out[k] = v
+	}
+	return nil
+}
+
 type matchedFeatures map[string]domainMatchedFeatures
 
 type domainMatchedFeatures map[string]interface{}
diff --git a/pkg/apis/nfd/v1alpha1/types.go b/pkg/apis/nfd/v1alpha1/types.go
index 2a6223eb6..d7bf89e7d 100644
--- a/pkg/apis/nfd/v1alpha1/types.go
+++ b/pkg/apis/nfd/v1alpha1/types.go
@@ -65,6 +65,19 @@ type Rule struct {
 	// +optional
 	LabelsTemplate string `json:"labelsTemplate"`
 
+	// Vars is the variables to store if the rule matches. Variables do not
+	// directly inflict any changes in the node object. However, they can be
+	// referenced from other rules enabling more complex rule hierarchies,
+	// without exposing intermediary output values as labels.
+	// +optional
+	Vars map[string]string `json:"vars"`
+
+	// VarsTemplate specifies a template to expand for dynamically generating
+	// multiple variables. Data (after template expansion) must be keys with an
+	// optional value (<key>[=<value>]) separated by newlines.
+	// +optional
+	VarsTemplate string `json:"varsTemplate"`
+
 	// MatchFeatures specifies a set of matcher terms all of which must match.
 	// +optional
 	MatchFeatures FeatureMatcher `json:"matchFeatures"`
@@ -177,3 +190,12 @@ const (
 	// expression must not have any values.
 	MatchIsFalse MatchOp = "IsFalse"
 )
+
+const (
+	// RuleBackrefDomain is the special feature domain for backreferencing
+	// output of preceding rules.
+	RuleBackrefDomain = "rule"
+	// RuleBackrefFeature is the special feature name for backreferencing
+	// output of preceding rules.
+	RuleBackrefFeature = "matched"
+)
diff --git a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go
index 95c87c0d3..ece20769a 100644
--- a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go
@@ -308,6 +308,13 @@ func (in *Rule) DeepCopyInto(out *Rule) {
 			(*out)[key] = val
 		}
 	}
+	if in.Vars != nil {
+		in, out := &in.Vars, &out.Vars
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.MatchFeatures != nil {
 		in, out := &in.MatchFeatures, &out.MatchFeatures
 		*out = make(FeatureMatcher, len(*in))
diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go
index bffbcce2e..519a4518a 100644
--- a/pkg/nfd-master/nfd-master.go
+++ b/pkg/nfd-master/nfd-master.go
@@ -43,7 +43,9 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 
+	"sigs.k8s.io/node-feature-discovery/pkg/api/feature"
 	"sigs.k8s.io/node-feature-discovery/pkg/apihelper"
+	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1"
 	pb "sigs.k8s.io/node-feature-discovery/pkg/labeler"
 	topologypb "sigs.k8s.io/node-feature-discovery/pkg/topologyupdater"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
@@ -499,6 +501,10 @@ func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string {
 
 	l := make(map[string]string)
 	ruleSpecs, err := m.nfdController.lister.List(labels.Everything())
+	sort.Slice(ruleSpecs, func(i, j int) bool {
+		return ruleSpecs[i].Name < ruleSpecs[j].Name
+	})
+
 	if err != nil {
 		klog.Errorf("failed to list LabelRule resources: %w", err)
 		return nil
@@ -519,10 +525,14 @@ func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string {
 				klog.Errorf("failed to process Rule %q: %w", rule.Name, err)
 				continue
 			}
-			for k, v := range ruleOut {
+
+			for k, v := range ruleOut.Labels {
 				l[k] = v
 			}
-			utils.KlogDump(1, "", "  ", ruleOut)
+
+			// Feed back rule output to features map for subsequent rules to match
+			feature.InsertFeatureValues(r.Features, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
+			feature.InsertFeatureValues(r.Features, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)
 		}
 	}
 
diff --git a/source/custom/custom.go b/source/custom/custom.go
index cde8d167c..48c0de50b 100644
--- a/source/custom/custom.go
+++ b/source/custom/custom.go
@@ -124,31 +124,34 @@ func (s *customSource) GetLabels() (source.FeatureLabels, error) {
 			continue
 		}
 
-		for n, v := range ruleOut {
+		for n, v := range ruleOut.Labels {
 			labels[n] = v
 		}
+		// Feed back rule output to features map for subsequent rules to match
+		feature.InsertFeatureValues(domainFeatures, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
+		feature.InsertFeatureValues(domainFeatures, nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)
 	}
 	return labels, nil
 }
 
-func (r *CustomRule) execute(features map[string]*feature.DomainFeatures) (map[string]string, error) {
+func (r *CustomRule) execute(features map[string]*feature.DomainFeatures) (nfdv1alpha1.RuleOutput, error) {
 	if r.LegacyRule != nil {
 		ruleOut, err := r.LegacyRule.execute(features)
 		if err != nil {
-			return nil, fmt.Errorf("failed to execute legacy rule %s: %w", r.LegacyRule.Name, err)
+			return nfdv1alpha1.RuleOutput{}, fmt.Errorf("failed to execute legacy rule %s: %w", r.LegacyRule.Name, err)
 		}
-		return ruleOut, err
+		return nfdv1alpha1.RuleOutput{Labels: ruleOut}, nil
 	}
 
 	if r.Rule != nil {
 		ruleOut, err := r.Rule.Execute(features)
 		if err != nil {
-			return nil, fmt.Errorf("failed to execute rule %s: %w", r.Rule.Name, err)
+			return ruleOut, fmt.Errorf("failed to execute rule %s: %w", r.Rule.Name, err)
 		}
-		return ruleOut, err
+		return ruleOut, nil
 	}
 
-	return nil, fmt.Errorf("BUG: an empty rule, this really should not happen")
+	return nfdv1alpha1.RuleOutput{}, fmt.Errorf("BUG: an empty rule, this really should not happen")
 }
 
 func (r *LegacyRule) execute(features map[string]*feature.DomainFeatures) (map[string]string, error) {


Test patch:
diff --git a/pkg/apis/nfd/v1alpha1/rule_test.go b/pkg/apis/nfd/v1alpha1/rule_test.go
index 0cadd0676..7a87c5da1 100644
--- a/pkg/apis/nfd/v1alpha1/rule_test.go
+++ b/pkg/apis/nfd/v1alpha1/rule_test.go
@@ -28,6 +28,7 @@ func TestRule(t *testing.T) {
 	r1 := Rule{Labels: map[string]string{"label-1": "", "label-2": "true"}}
 	r2 := Rule{
 		Labels: map[string]string{"label-1": "label-val-1"},
+		Vars:   map[string]string{"var-1": "var-val-1"},
 		MatchFeatures: FeatureMatcher{
 			FeatureMatcherTerm{
 				Feature: "domain-1.kf-1",
@@ -41,7 +42,7 @@ func TestRule(t *testing.T) {
 	// Test totally empty features
 	m, err := r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
 
 	_, err = r2.Execute(f)
 	assert.Error(t, err, "matching agains a missing domain should have returned an error")
@@ -52,7 +53,8 @@ func TestRule(t *testing.T) {
 
 	m, err = r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
+	assert.Empty(t, r1.Vars, "vars should be empty")
 
 	_, err = r2.Execute(f)
 	assert.Error(t, err, "matching agains a missing feature type should have returned an error")
@@ -64,11 +66,11 @@ func TestRule(t *testing.T) {
 
 	m, err = r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
 
 	m, err = r2.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "unexpected match")
+	assert.Nil(t, m.Labels, "unexpected match")
 
 	// Test non-empty feature sets
 	d.Keys["kf-1"].Elements["key-x"] = feature.Nil{}
@@ -78,17 +80,18 @@ func TestRule(t *testing.T) {
 
 	m, err = r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r1.Labels, m, "empty matcher should have matched empty features")
+	assert.Equal(t, r1.Labels, m.Labels, "empty matcher should have matched empty features")
 
 	// Match "key" features
 	m, err = r2.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "keys should not have matched")
+	assert.Nil(t, m.Labels, "keys should not have matched")
 
 	d.Keys["kf-1"].Elements["key-1"] = feature.Nil{}
 	m, err = r2.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r2.Labels, m, "keys should have matched")
+	assert.Equal(t, r2.Labels, m.Labels, "keys should have matched")
+	assert.Equal(t, r2.Vars, m.Vars, "vars should be present")
 
 	// Match "value" features
 	r3 := Rule{
@@ -104,12 +107,12 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r3.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "values should not have matched")
+	assert.Nil(t, m.Labels, "values should not have matched")
 
 	d.Values["vf-1"].Elements["key-1"] = "val-1"
 	m, err = r3.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r3.Labels, m, "values should have matched")
+	assert.Equal(t, r3.Labels, m.Labels, "values should have matched")
 
 	// Match "instance" features
 	r4 := Rule{
@@ -125,12 +128,12 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r4.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "instances should not have matched")
+	assert.Nil(t, m.Labels, "instances should not have matched")
 
 	d.Instances["if-1"].Elements[0].Attributes["attr-1"] = "val-1"
 	m, err = r4.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r4.Labels, m, "instances should have matched")
+	assert.Equal(t, r4.Labels, m.Labels, "instances should have matched")
 
 	// Test multiple feature matchers
 	r5 := Rule{
@@ -152,12 +155,12 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "instances should not have matched")
+	assert.Nil(t, m.Labels, "instances should not have matched")
 
 	r5.MatchFeatures[0].MatchExpressions.Expressions["key-1"] = MustCreateMatchExpression(MatchIn, "val-1")
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r5.Labels, m, "instances should have matched")
+	assert.Equal(t, r5.Labels, m.Labels, "instances should have matched")
 
 	// Test MatchAny
 	r5.MatchAny = []MatchAnyElem{
@@ -174,7 +177,7 @@ func TestRule(t *testing.T) {
 	}
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Nil(t, m, "instances should not have matched")
+	assert.Nil(t, m.Labels, "instances should not have matched")
 
 	r5.MatchAny = append(r5.MatchAny,
 		MatchAnyElem{
@@ -190,7 +193,7 @@ func TestRule(t *testing.T) {
 	r5.MatchFeatures[0].MatchExpressions.Expressions["key-1"] = MustCreateMatchExpression(MatchIn, "val-1")
 	m, err = r5.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, r5.Labels, m, "instances should have matched")
+	assert.Equal(t, r5.Labels, m.Labels, "instances should have matched")
 }
 
 func TestTemplating(t *testing.T) {
@@ -251,6 +254,12 @@ label-2=
 {{range .domain_1.vf_1}}vf-{{.Name}}=vf-{{.Value}}
 {{end}}
 {{range .domain_1.if_1}}if-{{index . "attr-1"}}_{{index . "attr-2"}}=present
+{{end}}`,
+		Vars: map[string]string{"var-1": "var-val-1"},
+		VarsTemplate: `
+var-1=value-will-be-overridden-by-vars
+var-2=
+{{range .domain_1.kf_1}}kf-{{.Name}}=true
 {{end}}`,
 		MatchFeatures: FeatureMatcher{
 			FeatureMatcherTerm{
@@ -294,10 +303,19 @@ label-2=
 		"if-1_val-2":   "present",
 		"if-10_val-20": "present",
 	}
+	expectedVars := map[string]string{
+		"var-1": "var-val-1",
+		"var-2": "",
+		// From template
+		"kf-key-a": "true",
+		"kf-key-c": "true",
+		"kf-foo":   "true",
+	}
 
 	m, err := r1.Execute(f)
 	assert.Nilf(t, err, "unexpected error: %v", err)
-	assert.Equal(t, expectedLabels, m, "instances should have matched")
+	assert.Equal(t, expectedLabels, m.Labels, "instances should have matched")
+	assert.Equal(t, expectedVars, m.Vars, "instances should have matched")
 
 	//
 	// Test error cases
@@ -316,7 +334,8 @@ label-2=
 	r2.LabelsTemplate = "foo=bar"
 	m, err = r2.Execute(f)
 	assert.Nil(t, err)
-	assert.Equal(t, map[string]string{"foo": "bar"}, m, "instances should have matched")
+	assert.Equal(t, map[string]string{"foo": "bar"}, m.Labels, "instances should have matched")
+	assert.Empty(t, m.Vars)
 
 	r2.labelsTemplate = nil
 	r2.LabelsTemplate = "foo"
@@ -327,4 +346,23 @@ label-2=
 	r2.LabelsTemplate = "{{"
 	_, err = r2.Execute(f)
 	assert.Error(t, err)
+
+	r2.labelsTemplate = nil
+	r2.LabelsTemplate = ""
+	r2.VarsTemplate = "bar=baz"
+	m, err = r2.Execute(f)
+	assert.Nil(t, err)
+	assert.Empty(t, m.Labels)
+	assert.Equal(t, map[string]string{"bar": "baz"}, m.Vars, "instances should have matched")
+
+	r2.varsTemplate = nil
+	r2.VarsTemplate = "bar"
+	_, err = r2.Execute(f)
+	assert.Error(t, err)
+
+	r2.varsTemplate = nil
+	r2.VarsTemplate = "{{"
+	_, err = r2.Execute(f)
+	assert.Error(t, err)
+
 }