func Add(mgr manager.Manager, c utils.Config) error {
	metrics.Registry.MustRegister(
		incarnationGitCreateLatency,
		incarnationGitDeployLatency,
		incarnationGitCanaryLatency,
		incarnationGitPendingReleaseLatency,
		incarnationGitReleaseLatency,
		incarnationRevisionDeployLatency,
		incarnationRevisionCanaryLatency,
		incarnationRevisionPendingReleaseLatency,
		incarnationRevisionReleaseLatency,
		incarnationRevisionRollbackLatency,
		incarnationDeployLatency,
		incarnationCanaryLatency,
		incarnationReleaseLatency,
		incarnationReleaseStateGauge,
		incarnationRevisionOldestStateGauge,
		revisionReleaseWeightGauge,
		reconcileInterval,
	)
	return add(mgr, newReconciler(mgr, c), c)
}
Add creates a new ReleaseManager Controller and adds it to the Manager. The Manager will set fields on the Controller
and Start it when the Manager is Started.
package releasemanager
import (
	"context"
	"fmt"
	"strconv"
	"time"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	picchuv1alpha1 "go.medium.engineering/picchu/pkg/apis/picchu/v1alpha1"
	"go.medium.engineering/picchu/pkg/controller/releasemanager/observe"
	"go.medium.engineering/picchu/pkg/controller/utils"
	"go.medium.engineering/picchu/pkg/plan"
	"github.com/go-logr/logr"
	"github.com/google/uuid"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"golang.org/x/sync/errgroup"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/controller-runtime/pkg/builder"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/event"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/metrics"
	"sigs.k8s.io/controller-runtime/pkg/predicate"
)
var (
	clog                        = logf.Log.WithName("controller_releasemanager")
	incarnationGitCreateLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_git_create_latency",
		Help:    "track time from git revision creation to incarnation create",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationGitDeployLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_git_deploy_latency",
		Help:    "track time from git revision creation to incarnation deploy",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationGitCanaryLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_git_canary_latency",
		Help:    "track time from git revision creation to incarnation canary",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationGitPendingReleaseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_git_pending_release_latency",
		Help:    "track time from git revision creation to incarnation pendingRelease",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationGitReleaseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_git_release_latency",
		Help:    "track time from git revision creation to incarnation release",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationRevisionDeployLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_revision_deploy_latency",
		Help:    "track time from revision creation to incarnation deploy",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationRevisionCanaryLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_revision_canary_latency",
		Help:    "track time from revision creation to incarnation canary",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationRevisionPendingReleaseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_revision_pending_release_latency",
		Help:    "track time from revision creation to incarnation pendingRelease",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	incarnationRevisionReleaseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_revision_release_latency",
		Help:    "track time from revision creation to incarnation release",
		Buckets: prometheus.ExponentialBuckets(120, 3, 7),
	}, []string{"app", "target"})
	revisionReleaseWeightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "picchu_revision_release_weight",
		Help: "Percent of traffic a revision is getting as a target release",
	}, []string{"app", "target"})
	incarnationRevisionRollbackLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_revision_rollback_latency",
		Help:    "track time from failed revision to rollbacked incarnation",
		Buckets: prometheus.ExponentialBuckets(30, 3, 7),
	}, []string{"app", "target"})
	incarnationReleaseStateGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "picchu_incarnation_count",
		Help: "Number of incarnations in a state",
	}, []string{"app", "target", "state"})
	incarnationRevisionOldestStateGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "picchu_revision_oldest_age_seconds",
		Help: "The oldest revision in seconds for each state",
	}, []string{"app", "target", "state"})
	incarnationDeployLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_deploy_latency",
		Help:    "track time a revision spends from deploying to deployed",
		Buckets: prometheus.ExponentialBuckets(30, 3, 7),
	}, []string{"app", "target"})
	incarnationCanaryLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_canary_latency",
		Help:    "track time a revision spends from canarying to canaried",
		Buckets: prometheus.ExponentialBuckets(30, 3, 7),
	}, []string{"app", "target"})
	incarnationReleaseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_release_latency",
		Help:    "track time a revision spends from releasing to released",
		Buckets: prometheus.ExponentialBuckets(30, 3, 7),
	}, []string{"app", "target"})
	reconcileInterval = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "picchu_release_manager_reconcile_interval",
		Help:    "track time between last update and reconcile request",
		Buckets: []float64{10, 15, 20, 30, 45, 60, 90, 120},
	}, []string{"app", "target"})
)