func (m *Module) runHooksByBindingAndCheckValues(binding BindingType, logLabels map[string]string) (bool, error) {
	var err error
	moduleHooks := m.moduleManager.GetModuleHooksInOrder(m.Name, binding)
	values, err := m.Values()
	if err != nil {
		return false, err
	}
	valuesChecksum, err := values.Checksum()
	if err != nil {
		return false, err
	}
	for _, moduleHookName := range moduleHooks {
		moduleHook := m.moduleManager.GetModuleHook(moduleHookName)
		err = moduleHook.RateLimitWait(context.Background())
		if err != nil {
			return false, err
		}
		bc := BindingContext{
			Binding: string(binding),
		}
		if binding == BeforeHelm || binding == AfterHelm || binding == AfterDeleteHelm {
			bc.Snapshots = moduleHook.HookController.KubernetesSnapshots()
			bc.Metadata.IncludeAllSnapshots = true
		}
		bc.Metadata.BindingType = binding
		metricLabels := map[string]string{
			"module":     m.Name,
			"hook":       moduleHook.Name,
			"binding":    string(binding),
			"queue":      "main",  
			"activation": logLabels["event.type"],
		}
		func() {
			defer measure.Duration(func(d time.Duration) {
				m.metricStorage.HistogramObserve("{PREFIX}module_hook_run_seconds", d.Seconds(), metricLabels, nil)
			})()
			err = moduleHook.Run(binding, []BindingContext{bc}, logLabels, metricLabels)
		}()
		if err != nil {
			return false, err
		}
	}
	newValues, err := m.Values()
	if err != nil {
		return false, err
	}
	newValuesChecksum, err := newValues.Checksum()
	if err != nil {
		return false, err
	}
	if newValuesChecksum != valuesChecksum {
		return true, nil
	}
	return false, nil
}
runHooksByBinding gets all hooks for binding, for each hook it creates a BindingContext,
sets KubernetesSnapshots and runs the hook. If values are changed after hooks execution, return true.
package module_manager
import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime/trace"
	"strings"
	"time"
	"github.com/flant/kube-client/manifest"
	"github.com/kennygrant/sanitize"
	log "github.com/sirupsen/logrus"
	uuid "gopkg.in/satori/go.uuid.v1"
	. "github.com/flant/addon-operator/pkg/hook/types"
	. "github.com/flant/shell-operator/pkg/hook/binding_context"
	. "github.com/flant/shell-operator/pkg/hook/types"
	sh_app "github.com/flant/shell-operator/pkg/app"
	"github.com/flant/shell-operator/pkg/executor"
	"github.com/flant/shell-operator/pkg/metric_storage"
	utils_file "github.com/flant/shell-operator/pkg/utils/file"
	"github.com/flant/shell-operator/pkg/utils/measure"
	"github.com/flant/addon-operator/pkg/app"
	"github.com/flant/addon-operator/pkg/helm"
	"github.com/flant/addon-operator/pkg/helm/client"
	"github.com/flant/addon-operator/pkg/utils"
	"github.com/flant/addon-operator/pkg/values/validation"
)
type Module struct {
	Name string
	Path string
	CommonStaticConfig *utils.ModuleConfig
	StaticConfig *utils.ModuleConfig
	LastReleaseManifests []manifest.Manifest
	State *ModuleState
	IsReady bool
	moduleManager *moduleManager
	metricStorage *metric_storage.MetricStorage
}
type ModuleState struct {
	OnStartupDone                bool
	SynchronizationTasksQueued   bool
	ShouldWaitForSynchronization bool
	WaitStarted                  bool
	SynchronizationDone bool
	MonitorsStarted bool
}
func NewModule(name, path string) *Module {
	return &Module{
		Name:  name,
		Path:  path,
		State: &ModuleState{},
	}
}
func (m *Module) WithModuleManager(moduleManager *moduleManager) {
	m.moduleManager = moduleManager
}
func (m *Module) WithMetricStorage(mstor *metric_storage.MetricStorage) {
	m.metricStorage = mstor
}
func (m *Module) SafeName() string {
	return sanitize.BaseName(m.Name)
}
func (m *Module) SynchronizationNeeded() bool {
	for _, hookName := range m.moduleManager.GetModuleHookNames(m.Name) {
		modHook := m.moduleManager.GetModuleHook(hookName)
		if modHook.SynchronizationNeeded() {
			return true
		}
	}
	return false
}
func (m *Module) SynchronizationQueued() bool {
	queued := false
	for _, hookName := range m.moduleManager.GetModuleHooksInOrder(m.Name, OnKubernetesEvent) {
		modHook := m.moduleManager.GetModuleHook(hookName)
		if modHook.SynchronizationQueued() {
			queued = true
		}
	}
	return queued
}
func (m *Module) SynchronizationDone() bool {
	done := true
	for _, hookName := range m.moduleManager.GetModuleHooksInOrder(m.Name, OnKubernetesEvent) {
		modHook := m.moduleManager.GetModuleHook(hookName)
		if modHook.SynchronizationNeeded() && !modHook.SynchronizationDone() {
			done = false
		}
	}
	return done
}
func (m *Module) RunOnStartup(logLabels map[string]string) error {
	logLabels = utils.MergeLabels(logLabels, map[string]string{
		"module": m.Name,
		"queue":  "main",
	})
	if err := m.cleanup(); err != nil {
		return err
	}
	if err := m.runHooksByBinding(OnStartup, logLabels); err != nil {
		return err
	}
	return nil
}
func (m *Module) Run(logLabels map[string]string) (bool, error) {
	defer trace.StartRegion(context.Background(), "ModuleRun-HelmPhase").End()
	logLabels = utils.MergeLabels(logLabels, map[string]string{
		"module": m.Name,
		"queue":  "main",
	})
	if err := m.cleanup(); err != nil {
		return false, err
	}
	m.moduleManager.HelmResourcesManager.PauseMonitor(m.Name)
	defer m.moduleManager.HelmResourcesManager.ResumeMonitor(m.Name)
	var err error
	treg := trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-beforeHelm")
	err = m.runHooksByBinding(BeforeHelm, logLabels)
	treg.End()
	if err != nil {
		return false, err
	}
	treg = trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-helm")
	err = m.runHelmInstall(logLabels)
	treg.End()
	if err != nil {
		return false, err
	}
	treg = trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-afterHelm")
	valuesChanged, err := m.runHooksByBindingAndCheckValues(AfterHelm, logLabels)
	treg.End()
	if err != nil {
		return false, err
	}
	return valuesChanged, nil
}
func (m *Module) Delete(logLabels map[string]string) error {
	defer trace.StartRegion(context.Background(), "ModuleDelete-HelmPhase").End()
	deleteLogLabels := utils.MergeLabels(logLabels,
		map[string]string{
			"module": m.Name,
			"queue":  "main",
		})
	logEntry := log.WithFields(utils.LabelsToLogFields(deleteLogLabels))
	m.moduleManager.HelmResourcesManager.StopMonitor(m.Name)
	chartExists, _ := m.checkHelmChart()
	if chartExists {
		releaseExists, err := helm.NewClient(deleteLogLabels).IsReleaseExists(m.generateHelmReleaseName())
		if !releaseExists {
			if err != nil {
				logEntry.Warnf("Cannot find helm release '%s' for module '%s'. Helm error: %s", m.generateHelmReleaseName(), m.Name, err)
			} else {
				logEntry.Warnf("Cannot find helm release '%s' for module '%s'.", m.generateHelmReleaseName(), m.Name)
			}
		} else {
			err := helm.NewClient(deleteLogLabels).DeleteRelease(m.generateHelmReleaseName())
			if err != nil {
				return err
			}
		}
	}
	return m.runHooksByBinding(AfterDeleteHelm, deleteLogLabels)
}
func (m *Module) cleanup() error {
	chartExists, err := m.checkHelmChart()
	if !chartExists {
		if err != nil {
			log.Debugf("MODULE '%s': cleanup is not needed: %s", m.Name, err)
			return nil
		}
	}
	helmLogLabels := map[string]string{
		"module": m.Name,
	}
	if err := helm.NewClient(helmLogLabels).DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil {
		return err
	}
	if err := helm.NewClient(helmLogLabels).DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil {
		return err
	}
	return nil
}
func (m *Module) runHelmInstall(logLabels map[string]string) error {
	metricLabels := map[string]string{
		"module":     m.Name,
		"activation": logLabels["event.type"],
	}
	defer measure.Duration(func(d time.Duration) {
		m.metricStorage.HistogramObserve("{PREFIX}module_helm_seconds", d.Seconds(), metricLabels, nil)
	})()
	logEntry := log.WithFields(utils.LabelsToLogFields(logLabels))
	chartExists, err := m.checkHelmChart()
	if !chartExists {
		if err != nil {
			logEntry.Debugf("no Chart.yaml, helm is not needed: %s", err)
			return nil
		}
	}
	err = m.checkHelmValues()
	if err != nil {
		return fmt.Errorf("check helm values: %v", err)
	}
	helmReleaseName := m.generateHelmReleaseName()
	valuesPath, err := m.PrepareValuesYamlFile()
	if err != nil {
		return err
	}
	defer os.Remove(valuesPath)
	helmClient := helm.NewClient(logLabels)
	var renderedManifests string
	func() {
		defer trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-helm-render").End()
		metricLabels := map[string]string{
			"module":     m.Name,
			"activation": logLabels["event.type"],
			"operation":  "template",
		}
		defer measure.Duration(func(d time.Duration) {
			m.metricStorage.HistogramObserve("{PREFIX}helm_operation_seconds", d.Seconds(), metricLabels, nil)
		})()
		renderedManifests, err = helmClient.Render(
			helmReleaseName,
			m.Path,
			[]string{valuesPath},
			[]string{},
			app.Namespace)
	}()
	if err != nil {
		return err
	}
	checksum := utils.CalculateStringsChecksum(renderedManifests)
	manifests, err := manifest.ListFromYamlDocs(renderedManifests)
	if err != nil {
		return err
	}
	logEntry.Debugf("chart has %d resources", len(manifests))
	m.LastReleaseManifests = manifests
	var runUpgradeRelease bool
	func() {
		defer trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-helm-check-upgrade").End()
		metricLabels := map[string]string{
			"module":     m.Name,
			"activation": logLabels["event.type"],
			"operation":  "check-upgrade",
		}
		defer measure.Duration(func(d time.Duration) {
			m.metricStorage.HistogramObserve("{PREFIX}helm_operation_seconds", d.Seconds(), metricLabels, nil)
		})()
		runUpgradeRelease, err = m.ShouldRunHelmUpgrade(helmClient, helmReleaseName, checksum, manifests, logLabels)
	}()
	if err != nil {
		return err
	}
	if !runUpgradeRelease {
		if !m.moduleManager.HelmResourcesManager.HasMonitor(m.Name) {
			m.moduleManager.HelmResourcesManager.StartMonitor(m.Name, manifests, app.Namespace)
		}
		return nil
	}
	func() {
		defer trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-helm-upgrade").End()
		metricLabels := map[string]string{
			"module":     m.Name,
			"activation": logLabels["event.type"],
			"operation":  "upgrade",
		}
		defer measure.Duration(func(d time.Duration) {
			m.metricStorage.HistogramObserve("{PREFIX}helm_operation_seconds", d.Seconds(), metricLabels, nil)
		})()
		err = helmClient.UpgradeRelease(
			helmReleaseName,
			m.Path,
			[]string{valuesPath},
			[]string{fmt.Sprintf("_addonOperatorModuleChecksum=%s", checksum)},
			app.Namespace,
		)
	}()
	if err != nil {
		return err
	}
	m.moduleManager.HelmResourcesManager.StartMonitor(m.Name, manifests, app.Namespace)
	return nil
}
func (m *Module) ShouldRunHelmUpgrade(helmClient client.HelmClient, releaseName string, checksum string, manifests []manifest.Manifest, logLabels map[string]string) (bool, error) {
	logEntry := log.WithFields(utils.LabelsToLogFields(logLabels))
	revision, status, err := helmClient.LastReleaseStatus(releaseName)
	if revision == "0" {
		logEntry.Debugf("helm release '%s' not exists: should run upgrade", releaseName)
		return true, nil
	}
	if err != nil {
		return false, err
	}
	if strings.ToLower(status) == "failed" {
		logEntry.Debugf("helm release '%s' has FAILED status: should run upgrade", releaseName)
		return true, nil
	}
	releaseValues, err := helmClient.GetReleaseValues(releaseName)
	if err != nil {
		logEntry.Debugf("helm release '%s' get values error, no upgrade: %v", releaseName, err)
		return false, err
	}
	recordedChecksum, hasKey := releaseValues["_addonOperatorModuleChecksum"]
	if !hasKey {
		logEntry.Debugf("helm release '%s' has no saved checksum of values: should run upgrade", releaseName)
		return true, nil
	}
	if recordedChecksumStr, ok := recordedChecksum.(string); ok {
		if recordedChecksumStr != checksum {
			logEntry.Debugf("helm release '%s' checksum '%s' is changed to '%s': should run upgrade", releaseName, recordedChecksumStr, checksum)
			return true, nil
		}
	}
	absent, err := m.moduleManager.HelmResourcesManager.GetAbsentResources(manifests, app.Namespace)
	if err != nil {
		return false, err
	}
	if len(absent) > 0 {
		logEntry.Debugf("helm release '%s' has %d absent resources: should run upgrade", releaseName, len(absent))
		return true, nil
	}
	logEntry.Debugf("helm release '%s' is unchanged: skip release upgrade", releaseName)
	return false, nil
}
func (m *Module) runHooksByBinding(binding BindingType, logLabels map[string]string) error {
	var err error
	moduleHooks := m.moduleManager.GetModuleHooksInOrder(m.Name, binding)
	for _, moduleHookName := range moduleHooks {
		moduleHook := m.moduleManager.GetModuleHook(moduleHookName)
		err = moduleHook.RateLimitWait(context.Background())
		if err != nil {
			return err
		}
		bc := BindingContext{
			Binding: string(binding),
		}
		if binding == BeforeHelm || binding == AfterHelm || binding == AfterDeleteHelm {
			bc.Snapshots = moduleHook.HookController.KubernetesSnapshots()
			bc.Metadata.IncludeAllSnapshots = true
		}
		bc.Metadata.BindingType = binding
		metricLabels := map[string]string{
			"module":     m.Name,
			"hook":       moduleHook.Name,
			"binding":    string(binding),
			"queue":      "main",  
			"activation": logLabels["event.type"],
		}
		func() {
			defer measure.Duration(func(d time.Duration) {
				m.metricStorage.HistogramObserve("{PREFIX}module_hook_run_seconds", d.Seconds(), metricLabels, nil)
			})()
			err = moduleHook.Run(binding, []BindingContext{bc}, logLabels, metricLabels)
		}()
		if err != nil {
			return err
		}
	}
	return nil
}