Repository: influxdata/telegraf-operator
Pull request #35 (2020-06-18 10:49:47)

feat: add support for monitoring istio sidecars

wojciechka: @gitirabassi I've applied the remaining changes that we wanted:

* [x] Add option to provide dedicated telegraf image for istio sidecar

* [x] Add `retentionPolicy: IfNeeded` to `dev.yml` file

* [x] Add checks for whether `telegraf` and/or `telegraf-istio` containers already exist

` [x] Add inputs for istio and only expect outputs in the istio class


Patch:
diff --git a/Dockerfile b/Dockerfile
index b095868..79f52b8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,7 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY main.go sidecar.go handler.go class_data.go ./
+COPY main.go sidecar.go handler.go class_data.go errors.go ./
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager *.go
diff --git a/class_data.go b/class_data.go
index 86f4e79..c1c9288 100644
--- a/class_data.go
+++ b/class_data.go
@@ -20,8 +20,6 @@ import (
 	"os"
 	"path/filepath"
 
-	corev1 "k8s.io/api/core/v1"
-
 	"github.com/go-logr/logr"
 	"github.com/influxdata/toml"
 )
@@ -29,7 +27,6 @@ import (
 type classDataHandler struct {
 	Logger                   logr.Logger
 	TelegrafClassesDirectory string
-	TelegrafDefaultClass     string
 }
 
 func (c *classDataHandler) validateClassData() error {
@@ -75,12 +72,7 @@ func (c *classDataHandler) validateClassData() error {
 	return nil
 }
 
-func (c *classDataHandler) getData(pod *corev1.Pod) (string, error) {
-	className := c.TelegrafDefaultClass
-	if extClass, ok := pod.Annotations[TelegrafClass]; ok {
-		className = extClass
-	}
-
+func (c *classDataHandler) getData(className string) (string, error) {
 	data, err := ioutil.ReadFile(filepath.Join(c.TelegrafClassesDirectory, className))
 
 	if err != nil {
diff --git a/deploy/dev.yml b/deploy/dev.yml
index 390e8d0..7f749c4 100644
--- a/deploy/dev.yml
+++ b/deploy/dev.yml
@@ -62,6 +62,7 @@ webhooks:
         apiVersions: ["*"]
         resources: ["pods"]
     failurePolicy: Ignore
+    reinvocationPolicy: IfNeeded
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -94,6 +95,9 @@ spec:
             # default class to use if not specified by the pod
             - --telegraf-default-class=basic
             - --telegraf-classes-directory=/config/classes
+            # allow injecting telegraf-istio sidecar for pods with
+            # istio sidecar annotations enabled
+            - --enable-istio-injection=true
           env:
             - name: POD_NAMESPACE
               valueFrom:
diff --git a/errors.go b/errors.go
new file mode 100644
index 0000000..674db11
--- /dev/null
+++ b/errors.go
@@ -0,0 +1,18 @@
+package main
+
+// error that notifies the handler that error occurred, but the pod should be created
+type nonFatalError struct {
+	err     error
+	message string
+}
+
+func newNonFatalError(err error, message string) error {
+	return &nonFatalError{
+		err:     err,
+		message: message,
+	}
+}
+
+func (e *nonFatalError) Error() string {
+	return e.err.Error()
+}
diff --git a/examples/classes.yml b/examples/classes.yml
index d08f431..cee2683 100644
--- a/examples/classes.yml
+++ b/examples/classes.yml
@@ -67,3 +67,7 @@ stringData:
       metric_batch_size = 100000
       metric_buffer_limit = 100000
       collection_jitter = "0s"
+  # example for gathering istio
+  istio: |+
+    [[outputs.influxdb]]
+      urls = ["http://influxdb.influxdb:8086"]
diff --git a/examples/nginx-with-istio.yml b/examples/nginx-with-istio.yml
new file mode 100644
index 0000000..9736fd4
--- /dev/null
+++ b/examples/nginx-with-istio.yml
@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-with-istio
+  namespace: test
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx-with-istio
+  template:
+    metadata:
+      labels:
+        app: nginx-with-istio
+      annotations:
+        # this will cause istio to inject its sidecar, that telegraf will then monitor
+        sidecar.istio.io/inject: "true"
+        # inject the status of the istio sidecar to mimic istio being present
+        sidecar.istio.io/status: "dummy"
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:alpine
diff --git a/handler.go b/handler.go
index 43b0d90..d448bc6 100644
--- a/handler.go
+++ b/handler.go
@@ -53,21 +53,29 @@ func (a *podInjector) Handle(ctx context.Context, req admission.Request) admissi
 	handlerLog.V(9).Info("request=" + string(marshaled))
 
 	if req.Operation == admv1.Delete {
-		secret := &corev1.Secret{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Secret",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      fmt.Sprintf("%s-%s", telegrafSecretPrefix, req.Name),
-				Namespace: req.Namespace,
-			},
+		deleteFailed := false
+		for _, name := range a.SidecarHandler.telegrafSecretNames(req.Name) {
+			secret := &corev1.Secret{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      name,
+					Namespace: req.Namespace,
+				},
+			}
+			handlerLog.Info("Deleting secret=" + secret.Name + "/" + secret.Namespace)
+			err := a.client.Delete(ctx, secret)
+			if err != nil {
+				handlerLog.Info("secret=" + secret.Name + "/" + secret.Namespace + " error:" + err.Error())
+				deleteFailed = true
+			}
 		}
-		err := a.client.Delete(ctx, secret)
-		if err != nil {
-			handlerLog.Info("secret=" + secret.Name + "/" + secret.Namespace + " error:" + err.Error())
-			return admission.Allowed("telegraf-injector coudn't delete secret")
+		if deleteFailed {
+			return admission.Allowed("telegraf-injector couldn't delete one or more secrets")
 		}
+
 		return admission.Allowed("telegraf-injector doesn't block pod deletions")
 	}
 
@@ -88,38 +96,42 @@ func (a *podInjector) Handle(ctx context.Context, req admission.Request) admissi
 		handlerLog.Info("name: " + name + ",  pod_getname=" + pod.GetName())
 	}
 
-	classData, err := a.ClassDataHandler.getData(pod)
-	if err != nil {
-		a.Logger.Info(fmt.Sprintf("unable to find class data: %v ; not adding sidecar container", err))
-		return admission.Allowed("telegraf-operator could not create sidecar container")
-	}
-
-	telegrafConf, err := a.SidecarHandler.assembleConf(pod, classData)
-	if err != nil {
-		a.Logger.Info(fmt.Sprintf("unable to assemble telegraf configuration: %v", err))
-		return admission.Allowed("telegraf-operator could not create sidecar container")
-	}
-
 	a.Logger.Info("adding sidecar container")
 	// if the telegraf configuration could be created, add sidecar pod
-	secret, err := a.SidecarHandler.addSidecar(pod, pod.GetName(), req.Namespace, telegrafConf)
+	result, err := a.SidecarHandler.addSidecars(pod, pod.GetName(), req.Namespace)
 	if err != nil {
-		a.Logger.Error(err, "unable to add sidecar container")
+
+		if nonFatalErr, ok := err.(*nonFatalError); ok {
+			a.Logger.Info(
+				fmt.Sprintf(
+					"unable to add telegraf sidecar container(s): %v ; not adding sidecar container, but allowing creation: %s",
+					nonFatalErr.err,
+					nonFatalErr.message,
+				),
+			)
+			return admission.Allowed(nonFatalErr.message)
+		}
+
+		a.Logger.Info(fmt.Sprintf("unable to add telegraf sidecar container(s): %v ; reporting error", err))
 		return admission.Errored(http.StatusBadRequest, err)
 	}
 
 	if req.Operation == admv1.Create {
-		err = a.client.Create(ctx, secret)
-		if err != nil {
-			a.Logger.Error(err, "unable to create secret")
-			return admission.Errored(http.StatusBadRequest, err)
+		for _, secret := range result.secrets {
+			err = a.client.Create(ctx, secret)
+			if err != nil {
+				a.Logger.Error(err, "unable to create secret")
+				return admission.Errored(http.StatusBadRequest, err)
+			}
 		}
 	}
 	if req.Operation == admv1.Update {
-		err = a.client.Update(ctx, secret)
-		if err != nil {
-			a.Logger.Error(err, "unable to update secret")
-			return admission.Errored(http.StatusBadRequest, err)
+		for _, secret := range result.secrets {
+			err = a.client.Update(ctx, secret)
+			if err != nil {
+				a.Logger.Error(err, "unable to update secret")
+				return admission.Errored(http.StatusBadRequest, err)
+			}
 		}
 	}
 
diff --git a/main.go b/main.go
index 5284557..d25cbaa 100644
--- a/main.go
+++ b/main.go
@@ -62,6 +62,9 @@ func main() {
 	var telegrafRequestsMemory string
 	var telegrafLimitsCPU string
 	var telegrafLimitsMemory string
+	var enableIstioInjection bool
+	var istioOutputClass string
+	var istioTelegrafImage string
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
@@ -75,6 +78,10 @@ func main() {
 	flag.StringVar(&telegrafRequestsMemory, "telegraf-requests-memory", defaultRequestsMemory, "Default requests for memory")
 	flag.StringVar(&telegrafLimitsCPU, "telegraf-limits-cpu", defaultLimitsCPU, "Default limits for CPU")
 	flag.StringVar(&telegrafLimitsMemory, "telegraf-limits-memory", defaultLimitsMemory, "Default limits for memory")
+	flag.BoolVar(&enableIstioInjection, "enable-istio-injection", false,
+		"Enable injecting additional sidecar for monitoring istio sidecar container. If enabled, additional sidecar telegraf-istio will be added for pods with the Istio annotation enabled")
+	flag.StringVar(&istioOutputClass, "istio-output-class", "istio", "Class to use for adding telegraf-istio sidecar to monitor its sidecar")
+	flag.StringVar(&istioTelegrafImage, "istio-telegraf-image", "", "If specified, use a custom image for telegraf-istio sidecar")
 	flag.Parse()
 
 	ctrl.SetLogger(zap.New(func(o *zap.Options) {
@@ -106,7 +113,6 @@ func main() {
 	classData := &classDataHandler{
 		Logger:                   logger,
 		TelegrafClassesDirectory: telegrafClassesDirectory,
-		TelegrafDefaultClass:     defaultTelegrafClass,
 	}
 
 	err = classData.validateClassData()
@@ -116,9 +122,14 @@ func main() {
 	}
 
 	sidecar := &sidecarHandler{
+		ClassDataHandler:            classData,
 		Logger:                      logger,
+		TelegrafDefaultClass:        defaultTelegrafClass,
 		TelegrafImage:               telegrafImage,
 		EnableDefaultInternalPlugin: enableDefaultInternalPlugin,
+		EnableIstioInjection:        enableIstioInjection,
+		IstioOutputClass:            istioOutputClass,
+		IstioTelegrafImage:          istioTelegrafImage,
 		RequestsCPU:                 telegrafRequestsCPU,
 		RequestsMemory:              telegrafRequestsMemory,
 		LimitsCPU:                   telegrafLimitsCPU,
diff --git a/sidecar.go b/sidecar.go
index b9140d6..2bf2d93 100644
--- a/sidecar.go
+++ b/sidecar.go
@@ -14,6 +14,16 @@ import (
 )
 
 const (
+	istioInputsConf = `
+  [[inputs.prometheus]]
+    urls = ["http://127.0.0.1:15090/stats/prometheus"]
+`
+)
+
+const (
+	// IstioSidecarAnnotation is the annotation used by istio sidecar handler
+	IstioSidecarAnnotation = "sidecar.istio.io/status"
+
 	// TelegrafAnnotationCommon is the shared prefix for all annotations.
 	TelegrafAnnotationCommon = "telegraf.influxdata.com"
 	// TelegrafMetricsPort is used to configure a port telegraf should scrape;
@@ -45,27 +55,64 @@ const (
 	TelegrafLimitsCPU = "telegraf.influxdata.com/limits-cpu"
 	// TelegrafLimitsMemory allows specifying custom memory resource limits
 	TelegrafLimitsMemory = "telegraf.influxdata.com/limits-memory"
-	telegrafSecretPrefix = "telegraf-config"
+	telegrafSecretInfix  = "config"
 )
 
 type sidecarHandler struct {
+	ClassDataHandler            *classDataHandler
 	Logger                      logr.Logger
+	TelegrafDefaultClass        string
 	TelegrafImage               string
 	EnableDefaultInternalPlugin bool
 	RequestsCPU                 string
 	RequestsMemory              string
 	LimitsCPU                   string
 	LimitsMemory                string
+	EnableIstioInjection        bool
+	IstioOutputClass            string
+	IstioTelegrafImage          string
+}
+
+type sidecarHandlerResponse struct {
+	// list of secrets to create alongside with the changes
+	secrets []*corev1.Secret
 }
 
 // This function check if the pod have the correct annotations, otherwise the controller will skip this pod entirely
 func (h *sidecarHandler) skip(pod *corev1.Pod) bool {
+	return !h.shouldAddTelegrafSidecar(pod) && !h.shouldAddIstioTelegrafSidecar(pod)
+}
+
+func (h *sidecarHandler) shouldAddTelegrafSidecar(pod *corev1.Pod) bool {
+	if podHasContainerName(pod, "telegraf") {
+		return false
+	}
+
 	for key := range pod.GetAnnotations() {
 		if strings.Contains(key, TelegrafAnnotationCommon) {
-			return false
+			return true
 		}
 	}
-	return true
+
+	return false
+}
+
+func (h *sidecarHandler) shouldAddIstioTelegrafSidecar(pod *corev1.Pod) bool {
+	if podHasContainerName(pod, "telegraf-istio") {
+		return false
+	}
+
+	if !h.EnableIstioInjection {
+		return false
+	}
+
+	for key := range pod.GetAnnotations() {
+		if key == IstioSidecarAnnotation {
+			return true
+		}
+	}
+
+	return false
 }
 
 func (h *sidecarHandler) validateRequestsAndLimits() error {
@@ -85,14 +132,81 @@ func (h *sidecarHandler) validateRequestsAndLimits() error {
 	return nil
 }
 
-func (h *sidecarHandler) addSidecar(pod *corev1.Pod, name, namespace, telegrafConf string) (*corev1.Secret, error) {
-	container, err := h.newContainer(pod)
+func (h *sidecarHandler) telegrafSecretNames(name string) []string {
+	return []string{
+		fmt.Sprintf("telegraf-%s-%s", telegrafSecretInfix, name),
+		fmt.Sprintf("telegraf-istio-%s-%s", telegrafSecretInfix, name),
+	}
+}
+
+func (h *sidecarHandler) addSidecars(pod *corev1.Pod, name, namespace string) (*sidecarHandlerResponse, error) {
+	result := &sidecarHandlerResponse{}
+	if h.shouldAddTelegrafSidecar(pod) {
+		err := h.addTelegrafSidecar(result, pod, name, namespace, "telegraf")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if h.shouldAddIstioTelegrafSidecar(pod) {
+		err := h.addIstioTelegrafSidecar(result, pod, name, namespace)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return result, nil
+}
+
+func (h *sidecarHandler) addTelegrafSidecar(result *sidecarHandlerResponse, pod *corev1.Pod, name, namespace, containerName string) error {
+	className := h.TelegrafDefaultClass
+	if extClass, ok := pod.Annotations[TelegrafClass]; ok {
+		className = extClass
+	}
+
+	classData, err := h.ClassDataHandler.getData(className)
 	if err != nil {
-		return nil, err
+		return newNonFatalError(err, "telegraf-operator could not create sidecar container for unknown class")
 	}
+
+	telegrafConf, err := h.assembleConf(pod, classData)
+	if err != nil {
+		return newNonFatalError(err, "telegraf-operator could not create sidecar container due to error in class data")
+	}
+
+	container, err := h.newContainer(pod, containerName)
+	if err != nil {
+		return err
+	}
+
+	return h.addContainerAndSecret(result, pod, container, name, namespace, telegrafConf)
+}
+
+func (h *sidecarHandler) addIstioTelegrafSidecar(result *sidecarHandlerResponse, pod *corev1.Pod, name, namespace string) error {
+	classData, err := h.ClassDataHandler.getData(h.IstioOutputClass)
+	if err != nil {
+		return newNonFatalError(err, "telegraf-operator could not create sidecar container for istio class")
+	}
+
+	telegrafConf := fmt.Sprintf("%s\n\n%s", istioInputsConf, classData)
+
+	container, err := h.newIstioContainer(pod, "telegraf-istio")
+	if err != nil {
+		return err
+	}
+
+	return h.addContainerAndSecret(result, pod, container, name, namespace, telegrafConf)
+}
+
+func (h *sidecarHandler) addContainerAndSecret(result *sidecarHandlerResponse, pod *corev1.Pod, container corev1.Container, name, namespace, telegrafConf string) error {
 	pod.Spec.Containers = append(pod.Spec.Containers, container)
-	pod.Spec.Volumes = append(pod.Spec.Volumes, h.newVolume(name))
-	return h.newSecret(pod, name, namespace, telegrafConf)
+	pod.Spec.Volumes = append(pod.Spec.Volumes, h.newVolume(name, container.Name))
+	secret, err := h.newSecret(pod, name, namespace, container.Name, telegrafConf)
+	if err != nil {
+		return err
+	}
+	result.secrets = append(result.secrets, secret)
+
+	return nil
 }
 
 // Assembling telegraf configuration
@@ -145,14 +259,14 @@ func (h *sidecarHandler) assembleConf(pod *corev1.Pod, classData string) (telegr
 	return telegrafConf, err
 }
 
-func (h *sidecarHandler) newSecret(pod *corev1.Pod, name, namespace, telegrafConf string) (*corev1.Secret, error) {
+func (h *sidecarHandler) newSecret(pod *corev1.Pod, name, namespace, containerName, telegrafConf string) (*corev1.Secret, error) {
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%s", telegrafSecretPrefix, name),
+			Name:      fmt.Sprintf("%s-%s-%s", containerName, telegrafSecretInfix, name),
 			Namespace: namespace,
 		},
 		Type: "Opaque",
@@ -162,12 +276,12 @@ func (h *sidecarHandler) newSecret(pod *corev1.Pod, name, namespace, telegrafCon
 	}, nil
 }
 
-func (h *sidecarHandler) newVolume(name string) corev1.Volume {
+func (h *sidecarHandler) newVolume(name, containerName string) corev1.Volume {
 	return corev1.Volume{
-		Name: "telegraf-config",
+		Name: fmt.Sprintf("%s-config", containerName),
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
-				SecretName: fmt.Sprintf("%s-%s", telegrafSecretPrefix, name),
+				SecretName: fmt.Sprintf("%s-%s-%s", containerName, telegrafSecretInfix, name),
 			},
 		},
 	}
@@ -183,7 +297,7 @@ func (h *sidecarHandler) parseCustomOrDefaultQuantity(customQuantity string, def
 	return quantity, err
 }
 
-func (h *sidecarHandler) newContainer(pod *corev1.Pod) (corev1.Container, error) {
+func (h *sidecarHandler) newContainer(pod *corev1.Pod, containerName string) (corev1.Container, error) {
 	var telegrafImage string
 	var telegrafRequestsCPU string
 	var telegrafRequestsMemory string
@@ -237,7 +351,7 @@ func (h *sidecarHandler) newContainer(pod *corev1.Pod) (corev1.Container, error)
 	}
 
 	baseContainer := corev1.Container{
-		Name:  "telegraf",
+		Name:  containerName,
 		Image: telegrafImage,
 		Resources: corev1.ResourceRequirements{
 			Limits: corev1.ResourceList{
@@ -262,7 +376,7 @@ func (h *sidecarHandler) newContainer(pod *corev1.Pod) (corev1.Container, error)
 
 		VolumeMounts: []corev1.VolumeMount{
 			{
-				Name:      "telegraf-config",
+				Name:      fmt.Sprintf("%s-config", containerName),
 				MountPath: "/etc/telegraf",
 			},
 		},
@@ -283,6 +397,67 @@ func (h *sidecarHandler) newContainer(pod *corev1.Pod) (corev1.Container, error)
 	return baseContainer, nil
 }
 
+func (h *sidecarHandler) newIstioContainer(pod *corev1.Pod, containerName string) (corev1.Container, error) {
+	var parsedRequestsCPU resource.Quantity
+	var parsedRequestsMemory resource.Quantity
+	var parsedLimitsCPU resource.Quantity
+	var parsedLimitsMemory resource.Quantity
+	var err error
+
+	if parsedRequestsCPU, err = resource.ParseQuantity(h.RequestsCPU); err != nil {
+		return corev1.Container{}, err
+	}
+	if parsedRequestsMemory, err = resource.ParseQuantity(h.RequestsMemory); err != nil {
+		return corev1.Container{}, err
+	}
+
+	if parsedLimitsCPU, err = resource.ParseQuantity(h.LimitsCPU); err != nil {
+		return corev1.Container{}, err
+	}
+	if parsedLimitsMemory, err = resource.ParseQuantity(h.LimitsMemory); err != nil {
+		return corev1.Container{}, err
+	}
+
+	telegrafImage := h.IstioTelegrafImage
+	if telegrafImage == "" {
+		telegrafImage = h.TelegrafImage
+	}
+
+	baseContainer := corev1.Container{
+		Name:  containerName,
+		Image: telegrafImage,
+		Resources: corev1.ResourceRequirements{
+			Limits: corev1.ResourceList{
+				"cpu":    parsedLimitsCPU,
+				"memory": parsedLimitsMemory,
+			},
+			Requests: corev1.ResourceList{
+				"cpu":    parsedRequestsCPU,
+				"memory": parsedRequestsMemory,
+			},
+		},
+		Env: []corev1.EnvVar{
+			{
+				Name: "NODENAME",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "spec.nodeName",
+					},
+				},
+			},
+		},
+
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      fmt.Sprintf("%s-config", containerName),
+				MountPath: "/etc/telegraf",
+			},
+		},
+	}
+
+	return baseContainer, nil
+}
+
 // ports gathers and merges unique ports from both TelegrafMetricsPort and TelegrafMetricsPorts.
 func ports(pod *corev1.Pod) []string {
 	uniquePorts := map[string]struct{}{}
@@ -305,3 +480,12 @@ func ports(pod *corev1.Pod) []string {
 	sort.Strings(ps)
 	return ps
 }
+
+func podHasContainerName(pod *corev1.Pod, name string) bool {
+	for _, container := range pod.Spec.Containers {
+		if container.Name == name {
+			return true
+		}
+	}
+	return false
+}


Test patch:
diff --git a/class_data_test.go b/class_data_test.go
index a8f681f..a6e5b0a 100644
--- a/class_data_test.go
+++ b/class_data_test.go
@@ -4,9 +4,6 @@ import (
 	"os"
 	"testing"
 
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	logrTesting "github.com/go-logr/logr/testing"
 )
 
@@ -24,7 +21,6 @@ func Test_classDataHandler_getData(t *testing.T) {
 		secretName string
 		className  string
 		namespace  string
-		pod        *corev1.Pod
 
 		want    string
 		wantErr bool
@@ -33,31 +29,18 @@ func Test_classDataHandler_getData(t *testing.T) {
 			name:      "data does not contain class name",
 			className: "unknown",
 			classes:   map[string]string{testTelegrafClass: sampleClassData},
-			pod:       &corev1.Pod{},
 			wantErr:   true,
 		},
 		{
 			name:      "returns secret data",
 			className: testTelegrafClass,
 			classes:   map[string]string{testTelegrafClass: sampleClassData},
-			pod:       &corev1.Pod{},
 			want:      sampleClassData,
 		},
-		{
-			name:    "returns secret data with annotation override",
-			classes: map[string]string{"name_override": sampleClassData},
-			pod: &corev1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{
-						TelegrafClass: "name_override",
-					},
-				},
-			},
-			want: sampleClassData,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+
 			logger := &logrTesting.TestLogger{T: t}
 
 			dir := createTempClassesDirectory(t, tt.classes)
@@ -66,10 +49,9 @@ func Test_classDataHandler_getData(t *testing.T) {
 			testClassDataHandler := &classDataHandler{
 				Logger:                   logger,
 				TelegrafClassesDirectory: dir,
-				TelegrafDefaultClass:     tt.className,
 			}
 
-			got, err := testClassDataHandler.getData(tt.pod)
+			got, err := testClassDataHandler.getData(tt.className)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("classDataHandler.getData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -81,7 +63,6 @@ func Test_classDataHandler_getData(t *testing.T) {
 	}
 }
 
-// TODO: test validateClassData
 func Test_classDataHandler_validateClassData(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -117,7 +98,6 @@ func Test_classDataHandler_validateClassData(t *testing.T) {
 			testClassDataHandler := &classDataHandler{
 				Logger:                   logger,
 				TelegrafClassesDirectory: dir,
-				TelegrafDefaultClass:     testTelegrafClass,
 			}
 
 			err := testClassDataHandler.validateClassData()
diff --git a/handler_test.go b/handler_test.go
index e7f0ce3..34ce4ed 100644
--- a/handler_test.go
+++ b/handler_test.go
@@ -548,6 +548,157 @@ func Test_podInjector_Handle(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "skip telegraf into container if istio annotation present, but option enabled",
+			req: admission.Request{
+				AdmissionRequest: admv1.AdmissionRequest{
+					Operation: admv1.Create,
+					Object: runtime.RawExtension{
+						Raw: []byte(`{
+								"apiVersion": "v1",
+								"kind": "Pod",
+								"metadata": {
+								  "name": "simple",
+								  "annotations": {
+									"sidecar.istio.io/status": "dummy"
+								  }
+								},
+								"spec": {
+								  "containers": [
+									{
+									  "name": "busybox",
+									  "image": "busybox",
+									  "args": [
+										"sleep",
+										"1000000"
+									  ]
+									}
+								  ]
+								}
+							  }`),
+					},
+				},
+			},
+			fields: fields{
+				TelegrafDefaultClass: testTelegrafClass,
+			},
+			classes: map[string]string{testTelegrafClass: sampleClassData},
+			want: want{
+				Allowed: true,
+			},
+		},
+		{
+			name: "inject telegraf-istio into container if option enabled",
+			req: admission.Request{
+				AdmissionRequest: admv1.AdmissionRequest{
+					Operation: admv1.Create,
+					Object: runtime.RawExtension{
+						Raw: []byte(`{
+								"apiVersion": "v1",
+								"kind": "Pod",
+								"metadata": {
+								  "name": "simple",
+								  "annotations": {
+									"sidecar.istio.io/status": "dummy"
+								  }
+								},
+								"spec": {
+								  "containers": [
+									{
+									  "name": "busybox",
+									  "image": "busybox",
+									  "args": [
+										"sleep",
+										"1000000"
+									  ]
+									}
+								  ]
+								}
+							  }`),
+					},
+				},
+			},
+			fields: fields{
+				TelegrafDefaultClass: testTelegrafClass,
+			},
+			handler: &sidecarHandler{
+				RequestsCPU:          defaultRequestsCPU,
+				RequestsMemory:       defaultRequestsMemory,
+				LimitsCPU:            defaultLimitsCPU,
+				LimitsMemory:         defaultLimitsMemory,
+				EnableIstioInjection: true,
+				IstioOutputClass:     testTelegrafClass,
+			},
+			classes: map[string]string{testTelegrafClass: sampleClassData},
+			want: want{
+				Allowed: true,
+				Patches: []string{
+					`{"op":"add","path":"/metadata/creationTimestamp"}`,
+					`{"op":"add","path":"/spec/containers/0/resources","value":{}}`,
+					`{"op":"add","path":"/spec/containers/1","value":{"env":[{"name":"NODENAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"image":"docker.io/library/telegraf:1.13","name":"telegraf-istio","resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"50m","memory":"50Mi"}},"volumeMounts":[{"mountPath":"/etc/telegraf","name":"telegraf-istio-config"}]}}`,
+					`{"op":"add","path":"/spec/volumes","value":[{"name":"telegraf-istio-config","secret":{"secretName":"telegraf-istio-config-simple"}}]}`,
+					`{"op":"add","path":"/status","value":{}}`,
+				},
+			},
+		},
+		{
+			name: "inject telegraf and telegraf-istio into container",
+			req: admission.Request{
+				AdmissionRequest: admv1.AdmissionRequest{
+					Operation: admv1.Create,
+					Object: runtime.RawExtension{
+						Raw: []byte(`{
+								"apiVersion": "v1",
+								"kind": "Pod",
+								"metadata": {
+								  "name": "simple",
+								  "annotations": {
+									"sidecar.istio.io/status": "dummy",
+									"telegraf.influxdata.com/port": "8080",
+									"telegraf.influxdata.com/path": "/v1/metrics",
+									"telegraf.influxdata.com/interval": "5s"
+								  }
+								},
+								"spec": {
+								  "containers": [
+									{
+									  "name": "busybox",
+									  "image": "busybox",
+									  "args": [
+										"sleep",
+										"1000000"
+									  ]
+									}
+								  ]
+								}
+							  }`),
+					},
+				},
+			},
+			fields: fields{
+				TelegrafDefaultClass: testTelegrafClass,
+			},
+			handler: &sidecarHandler{
+				RequestsCPU:          defaultRequestsCPU,
+				RequestsMemory:       defaultRequestsMemory,
+				LimitsCPU:            defaultLimitsCPU,
+				LimitsMemory:         defaultLimitsMemory,
+				EnableIstioInjection: true,
+				IstioOutputClass:     testTelegrafClass,
+			},
+			classes: map[string]string{testTelegrafClass: sampleClassData},
+			want: want{
+				Allowed: true,
+				Patches: []string{
+					`{"op":"add","path":"/metadata/creationTimestamp"}`,
+					`{"op":"add","path":"/spec/containers/0/resources","value":{}}`,
+					`{"op":"add","path":"/spec/containers/1","value":{"env":[{"name":"NODENAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"image":"docker.io/library/telegraf:1.13","name":"telegraf","resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"50m","memory":"50Mi"}},"volumeMounts":[{"mountPath":"/etc/telegraf","name":"telegraf-config"}]}}`,
+					`{"op":"add","path":"/spec/containers/2","value":{"env":[{"name":"NODENAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"image":"docker.io/library/telegraf:1.13","name":"telegraf-istio","resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"50m","memory":"50Mi"}},"volumeMounts":[{"mountPath":"/etc/telegraf","name":"telegraf-istio-config"}]}}`,
+					`{"op":"add","path":"/spec/volumes","value":[{"name":"telegraf-config","secret":{"secretName":"telegraf-config-simple"}},{"name":"telegraf-istio-config","secret":{"secretName":"telegraf-istio-config-simple"}}]}`,
+					`{"op":"add","path":"/status","value":{}}`,
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -582,9 +733,11 @@ func Test_podInjector_Handle(t *testing.T) {
 			testClassDataHandler := &classDataHandler{
 				Logger:                   logger,
 				TelegrafClassesDirectory: dir,
-				TelegrafDefaultClass:     tt.fields.TelegrafDefaultClass,
 			}
 
+			tt.handler.ClassDataHandler = testClassDataHandler
+			tt.handler.TelegrafDefaultClass = tt.fields.TelegrafDefaultClass
+
 			p := &podInjector{
 				client:           client,
 				decoder:          decoder,
@@ -593,6 +746,10 @@ func Test_podInjector_Handle(t *testing.T) {
 				ClassDataHandler: testClassDataHandler,
 			}
 
+			if tt.want.Code == 0 {
+				tt.want.Code = http.StatusOK
+			}
+
 			resp := p.Handle(context.Background(), tt.req)
 
 			if got, want := resp.Allowed, tt.want.Allowed; got != want {
diff --git a/sidecar_test.go b/sidecar_test.go
index 5d9b8a8..53d3447 100644
--- a/sidecar_test.go
+++ b/sidecar_test.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -13,6 +14,36 @@ import (
 	encode "k8s.io/apimachinery/pkg/runtime/serializer/json"
 )
 
+var (
+	testEmptySecret = `
+apiVersion: v1
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: telegraf-config-myname
+  namespace: mynamespace
+stringData:
+  telegraf.conf: |2+
+
+type: Opaque`
+	testEmptyIstioSecret = `
+apiVersion: v1
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: telegraf-istio-config-myname
+  namespace: mynamespace
+stringData:
+  telegraf.conf: |2-
+
+      [[inputs.prometheus]]
+        urls = ["http://127.0.0.1:15090/stats/prometheus"]
+
+
+    # istio outputs
+type: Opaque`
+)
+
 func Test_skip(t *testing.T) {
 	handler := &sidecarHandler{
 		RequestsCPU:    defaultRequestsCPU,
@@ -205,7 +236,9 @@ func Test_assembleConf(t *testing.T) {
 			enableDefaultInternalPlugin: true,
 			pod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						TelegrafClass: "default",
+					},
 				},
 			},
 			wantConfig: `
@@ -236,12 +269,15 @@ func Test_assembleConf(t *testing.T) {
 	}
 }
 
-func Test_addSidecar(t *testing.T) {
+func Test_addSidecars(t *testing.T) {
 	tests := []struct {
 		name                        string
 		pod                         *corev1.Pod
 		enableDefaultInternalPlugin bool
-		wantSecret                  string
+		enableIstioInjection        bool
+		istioTelegrafImage          string
+		istioOutputClass            string
+		wantSecrets                 []string
 		wantPod                     string
 	}{
 		{
@@ -253,7 +289,8 @@ func Test_addSidecar(t *testing.T) {
 					},
 				},
 			},
-			wantSecret: `apiVersion: v1
+			wantSecrets: []string{
+				`apiVersion: v1
 kind: Secret
 metadata:
   creationTimestamp: null
@@ -263,16 +300,21 @@ stringData:
   telegraf.conf: "\n[[inputs.prometheus]]\n  urls = [\"http://127.0.0.1:6060/metrics\"]\n
     \ \n\n"
 type: Opaque`,
+			},
 		},
 		{
 			name: "validate default telegraf pod definition",
 			pod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						TelegrafClass: "default",
+					},
 				},
 			},
 			wantPod: `
 metadata:
+  annotations:
+    telegraf.influxdata.com/class: default
   creationTimestamp: null
 spec:
   containers:
@@ -299,6 +341,7 @@ spec:
       secretName: telegraf-config-myname
 status: {}
 			`,
+			wantSecrets: []string{testEmptySecret},
 		},
 		{
 			name: "validate custom telegraf image pod definition",
@@ -339,14 +382,20 @@ spec:
       secretName: telegraf-config-myname
 status: {}
 			`,
+			wantSecrets: []string{testEmptySecret},
 		},
 		{
 			name: "validate enable default internal plugin",
 			pod: &corev1.Pod{
-				ObjectMeta: metav1.ObjectMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						TelegrafClass: "default",
+					},
+				},
 			},
 			enableDefaultInternalPlugin: true,
-			wantSecret: `apiVersion: v1
+			wantSecrets: []string{
+				`apiVersion: v1
 kind: Secret
 metadata:
   creationTimestamp: null
@@ -358,6 +407,7 @@ stringData:
     [[inputs.internal]]
 
 type: Opaque`,
+			},
 		},
 		{
 			name: "validate custom resources and limits",
@@ -404,6 +454,50 @@ spec:
       secretName: telegraf-config-myname
 status: {}
 			`,
+			wantSecrets: []string{testEmptySecret},
+		},
+		{
+			name: "validate incorrect resources to fallback default resources",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						TelegrafRequestsCPU: "100x",
+						TelegrafLimitsCPU:   "750m",
+					},
+				},
+			},
+			wantPod: `
+metadata:
+  annotations:
+    telegraf.influxdata.com/limits-cpu: 750m
+    telegraf.influxdata.com/requests-cpu: 100x
+  creationTimestamp: null
+spec:
+  containers:
+  - env:
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    image: docker.io/library/telegraf:1.13
+    name: telegraf
+    resources:
+      limits:
+        cpu: 750m
+        memory: 500Mi
+      requests:
+        cpu: 50m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/telegraf
+      name: telegraf-config
+  volumes:
+  - name: telegraf-config
+    secret:
+      secretName: telegraf-config-myname
+status: {}
+`,
+			wantSecrets: []string{testEmptySecret},
 		},
 		{
 			name: "validate incorrect resources to fallback default resources",
@@ -446,32 +540,287 @@ spec:
       secretName: telegraf-config-myname
 status: {}
 `,
+			wantSecrets: []string{testEmptySecret},
+		},
+		{
+			name: "does not add telegraf sidecar when container already exists",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						TelegrafClass: "default",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						corev1.Container{
+							Name:  "telegraf",
+							Image: "alpine:latest",
+						},
+					},
+				},
+			},
+			wantPod: `
+metadata:
+  annotations:
+    telegraf.influxdata.com/class: default
+  creationTimestamp: null
+spec:
+  containers:
+  - image: alpine:latest
+    name: telegraf
+    resources: {}
+status: {}
+`,
+			wantSecrets: []string{},
+		},
+		{
+			name: "does not add istio sidecar when not enabled",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						IstioSidecarAnnotation: "dummy",
+					},
+				},
+			},
+			wantPod: `
+metadata:
+  annotations:
+    sidecar.istio.io/status: dummy
+  creationTimestamp: null
+spec:
+  containers: null
+status: {}
+`,
+			wantSecrets: []string{},
+		},
+		{
+			name: "adds istio sidecar when sidecar annotation enabled and istio injection enabled",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						IstioSidecarAnnotation: "dummy",
+					},
+				},
+			},
+			enableIstioInjection: true,
+			istioOutputClass:     "istio",
+			wantPod: `
+metadata:
+  annotations:
+    sidecar.istio.io/status: dummy
+  creationTimestamp: null
+spec:
+  containers:
+  - env:
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    image: docker.io/library/telegraf:1.13
+    name: telegraf-istio
+    resources:
+      limits:
+        cpu: 500m
+        memory: 500Mi
+      requests:
+        cpu: 50m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/telegraf
+      name: telegraf-istio-config
+  volumes:
+  - name: telegraf-istio-config
+    secret:
+      secretName: telegraf-istio-config-myname
+status: {}
+`,
+			wantSecrets: []string{testEmptyIstioSecret},
+		},
+		{
+			name: "does not add istio sidecar when container already exists",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						IstioSidecarAnnotation: "dummy",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						corev1.Container{
+							Name:  "telegraf-istio",
+							Image: "alpine:latest",
+						},
+					},
+				},
+			},
+			enableIstioInjection: true,
+			istioOutputClass:     "istio",
+			wantPod: `
+metadata:
+  annotations:
+    sidecar.istio.io/status: dummy
+  creationTimestamp: null
+spec:
+  containers:
+  - image: alpine:latest
+    name: telegraf-istio
+    resources: {}
+status: {}
+`,
+			wantSecrets: []string{},
+		},
+		{
+			name: "adds istio sidecar when sidecar annotation enabled and istio injection enabled",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						IstioSidecarAnnotation: "dummy",
+					},
+				},
+			},
+			enableIstioInjection: true,
+			istioTelegrafImage:   "docker.io/library/telegraf:1.11",
+			istioOutputClass:     "istio",
+			wantPod: `
+metadata:
+  annotations:
+    sidecar.istio.io/status: dummy
+  creationTimestamp: null
+spec:
+  containers:
+  - env:
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    image: docker.io/library/telegraf:1.11
+    name: telegraf-istio
+    resources:
+      limits:
+        cpu: 500m
+        memory: 500Mi
+      requests:
+        cpu: 50m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/telegraf
+      name: telegraf-istio-config
+  volumes:
+  - name: telegraf-istio-config
+    secret:
+      secretName: telegraf-istio-config-myname
+status: {}
+`,
+			wantSecrets: []string{testEmptyIstioSecret},
+		},
+		{
+			name: "adds both regular telegraf and istio sidecars when sidecar annotation enabled and istio injection enabled",
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						IstioSidecarAnnotation: "dummy",
+						TelegrafClass:          "default",
+					},
+				},
+			},
+			enableIstioInjection: true,
+			istioOutputClass:     "istio",
+			wantPod: `
+metadata:
+  annotations:
+    sidecar.istio.io/status: dummy
+    telegraf.influxdata.com/class: default
+  creationTimestamp: null
+spec:
+  containers:
+  - env:
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    image: docker.io/library/telegraf:1.13
+    name: telegraf
+    resources:
+      limits:
+        cpu: 500m
+        memory: 500Mi
+      requests:
+        cpu: 50m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/telegraf
+      name: telegraf-config
+  - env:
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    image: docker.io/library/telegraf:1.13
+    name: telegraf-istio
+    resources:
+      limits:
+        cpu: 500m
+        memory: 500Mi
+      requests:
+        cpu: 50m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/telegraf
+      name: telegraf-istio-config
+  volumes:
+  - name: telegraf-config
+    secret:
+      secretName: telegraf-config-myname
+  - name: telegraf-istio-config
+    secret:
+      secretName: telegraf-istio-config-myname
+status: {}
+`,
+			wantSecrets: []string{testEmptySecret, testEmptyIstioSecret},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			dir := createTempClassesDirectory(t, map[string]string{
+				"default": "",
+				"istio":   "# istio outputs",
+			})
+			defer os.RemoveAll(dir)
+
+			logger := &logrTesting.TestLogger{T: t}
+
+			testClassDataHandler := &classDataHandler{
+				Logger:                   logger,
+				TelegrafClassesDirectory: dir,
+			}
+
 			handler := &sidecarHandler{
+				ClassDataHandler:            testClassDataHandler,
+				TelegrafDefaultClass:        "default",
 				TelegrafImage:               defaultTelegrafImage,
 				EnableDefaultInternalPlugin: tt.enableDefaultInternalPlugin,
+				EnableIstioInjection:        tt.enableIstioInjection,
+				IstioOutputClass:            tt.istioOutputClass,
+				IstioTelegrafImage:          tt.istioTelegrafImage,
 				RequestsCPU:                 defaultRequestsCPU,
 				RequestsMemory:              defaultRequestsMemory,
 				LimitsCPU:                   defaultLimitsCPU,
 				LimitsMemory:                defaultLimitsMemory,
 				Logger:                      &logrTesting.TestLogger{T: t},
 			}
-			telegrafConf, err := handler.assembleConf(tt.pod, "")
-			if err != nil {
-				t.Errorf("unexpected error assembling sidecar configuration: %v", err)
-			}
-			secret, err := handler.addSidecar(tt.pod, "myname", "mynamespace", telegrafConf)
+
+			result, err := handler.addSidecars(tt.pod, "myname", "mynamespace")
 			if err != nil {
 				t.Errorf("unexpected error adding to sidecar: %v", err)
 			}
 
-			if tt.wantSecret != "" {
-				if want, got := strings.TrimSpace(tt.wantSecret), strings.TrimSpace(toYAML(t, secret)); got != want {
-					t.Errorf("unexpected secret got:\n%v\nwant:\n%v", got, want)
+			if want, got := len(tt.wantSecrets), len(result.secrets); got != want {
+				t.Errorf("invalid number of secrets returned got: %d; want: %d", got, want)
+			}
+			for i := 0; i < len(result.secrets); i++ {
+				if want, got := strings.TrimSpace(tt.wantSecrets[i]), strings.TrimSpace(toYAML(t, result.secrets[i])); got != want {
+					t.Errorf("unexpected secret %d got:\n%v\nwant:\n%v", i, got, want)
 				}
 			}