Repository: jenkins-x/lighthouse
Pull request #897 (2020-08-10 09:05:53)

fix: add a new repoconfig package and file format
so that we can implement different strategies for configuring lighthouse such as:

* a git repository with global configuration using traditional prow/lh config and then a directory tree of repository specific configurations
* detect lighthouse configuration inside the actual git repositories themselves; so git repositories can configure their own presubmits/postsubmits like the GitHub Actions model

here is an example [lighthouse.yaml](https://github.com/jenkins-x/lighthouse/blob/repo-config/pkg/repoconfig/merge/test_data/simple/lighthouse.yaml) file to get the idea. The current global configuration for lighthouse is pretty big, complex and easy to get wrong - so the idea of this repository specific configuration is to make a single, simpler YAML format folks can use if they are not too worried about sticking with the classic global prow configuration format.

This new config file format aims to allow 1 repository to be configured in a smaller simpler configuration file using versioning + CRD-like yaml so it feels more natural to kubernetes developers.

Note that this PR doesn't actually change how lighthouse loads its configuration yet; its more just defining a repositories configuration file and helper methods for merging the repository-specific configuration file(s) with the global configuration.

In future PRs we can try introduce a strategy/interface for getting the config/plugins configuration for a repository and integrating it into the current lighthouse code so we can maybe have a couple of strategies for loading configuration (e.g. loading from a single git repository, loading from ConfigMaps or a global configuration + repository specific configurations inside git etc)


Patch:
diff --git a/pkg/apis/lighthouse/v1alpha1/repoconfig_config_types.go b/pkg/apis/lighthouse/v1alpha1/repoconfig_config_types.go
new file mode 100644
index 00000000..23cba0d2
--- /dev/null
+++ b/pkg/apis/lighthouse/v1alpha1/repoconfig_config_types.go
@@ -0,0 +1,182 @@
+package v1alpha1
+
+import (
+	"regexp"
+
+	tektonv1beta1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
+	v1 "k8s.io/api/core/v1"
+)
+
+// These structs are copies of the config package structs but with nicer kubernetes native camel case
+// yaml encoding
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import (
+	"time"
+)
+
+// this file contains copies of structs from the config package
+// using different JSON/YAML marshalling so that they look like
+// kubernetes CRDs; using camelCase rather than lower case and underscores
+
+// JobBase contains attributes common to all job types
+type JobBase struct {
+	// The name of the job. Must match regex [A-Za-z0-9-._]+
+	// e.g. pull-test-infra-bazel-build
+	Name string `json:"name"`
+	// Labels are added to LighthouseJobs and pods created for this job.
+	Labels map[string]string `json:"labels,omitempty"`
+	// Annotations are unused by prow itself, but provide a space to configure other automation.
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// MaximumConcurrency of this job, 0 implies no limit.
+	MaxConcurrency int `json:"maxConcurrency,omitempty"`
+	// Agent that will take care of running this job.
+	Agent string `json:"agent"`
+	// Cluster is the alias of the cluster to run this job in.
+	// (Default: kube.DefaultClusterAlias)
+	Cluster string `json:"cluster,omitempty"`
+	// Namespace is the namespace in which pods schedule.
+	//   nil: results in config.PodNamespace (aka pod default)
+	//   empty: results in config.LighthouseJobNamespace (aka same as LighthouseJob)
+	Namespace *string `json:"namespace,omitempty"`
+	// ErrorOnEviction indicates that the LighthouseJob should be completed and given
+	// the ErrorState status if the pod that is executing the job is evicted.
+	// If this field is unspecified or false, a new pod will be created to replace
+	// the evicted one.
+	ErrorOnEviction bool `json:"errorOnEviction,omitempty"`
+	// SourcePath contains the path where this job is defined
+	SourcePath string `json:"-"`
+	// Spec is the Kubernetes pod spec used if Agent is kubernetes.
+	Spec *v1.PodSpec `json:"spec,omitempty"`
+	// PipelineRunSpec is the Tekton PipelineRun spec used if agent is tekton-pipeline
+	PipelineRunSpec *tektonv1beta1.PipelineRunSpec `json:"pipelineRunSpec,omitempty"`
+
+	UtilityConfig
+}
+
+// Presubmit runs on PRs.
+type Presubmit struct {
+	JobBase
+
+	// AlwaysRun automatically for every PR, or only when a comment triggers it.
+	AlwaysRun bool `json:"alwaysRun"`
+
+	// Optional indicates that the job's status context should not be required for merge.
+	Optional bool `json:"optional,omitempty"`
+
+	// Trigger is the regular expression to trigger the job.
+	// e.g. `@k8s-bot e2e test this`
+	// RerunCommand must also be specified if this field is specified.
+	// (Default: `(?m)^/test (?:.*? )?<job name>(?: .*?)?$`)
+	Trigger string `json:"trigger,omitempty"`
+
+	// The RerunCommand to give users. Must match Trigger.
+	// Trigger must also be specified if this field is specified.
+	// (Default: `/test <job name>`)
+	RerunCommand string `json:"rerunCommand,omitempty"`
+
+	Brancher
+
+	RegexpChangeMatcher
+
+	Reporter
+
+	// We'll set these when we load it.
+	//re *regexp.Regexp // from Trigger.
+}
+
+// Postsubmit runs on push events.
+type Postsubmit struct {
+	JobBase
+
+	RegexpChangeMatcher
+
+	Brancher
+
+	// TODO(krzyzacy): Move existing `Report` into `SkipReport` once this is deployed
+	Reporter
+}
+
+// Periodic runs on a timer.
+type Periodic struct {
+	JobBase
+
+	// (deprecated)Interval to wait between two runs of the job.
+	Interval string `json:"interval"`
+	// Cron representation of job trigger time
+	Cron string `json:"cron"`
+	// Tags for config entries
+	Tags []string `json:"tags,omitempty"`
+
+	interval time.Duration
+}
+
+// Brancher is for shared code between jobs that only run against certain
+// branches. An empty brancher runs against all branches.
+type Brancher struct {
+	// Do not run against these branches. Default is no branches.
+	SkipBranches []string `json:"skipBranches,omitempty"`
+	// Only run against these branches. Default is all branches.
+	Branches []string `json:"branches,omitempty"`
+
+	// We'll set these when we load it.
+	re     *regexp.Regexp
+	reSkip *regexp.Regexp
+}
+
+// RegexpChangeMatcher is for code shared between jobs that run only when certain files are changed.
+type RegexpChangeMatcher struct {
+	// RunIfChanged defines a regex used to select which subset of file changes should trigger this job.
+	// If any file in the changeset matches this regex, the job will be triggered
+	RunIfChanged string         `json:"runIfChanged,omitempty"`
+	reChanges    *regexp.Regexp // from RunIfChanged
+}
+
+// Reporter keeps various details for status reporting
+type Reporter struct {
+	// Context is the name of the GitHub status context for the job.
+	// Defaults: the same as the name of the job.
+	Context string `json:"context,omitempty"`
+	// SkipReport skips commenting and setting status on GitHub.
+	SkipReport bool `json:"skipReport,omitempty"`
+}
+
+// ChangedFilesProvider returns a slice of modified files.
+type ChangedFilesProvider func() ([]string, error)
+
+// UtilityConfig holds decoration metadata, such as how to clone and additional containers/etc
+type UtilityConfig struct {
+	// Decorate determines if we decorate the PodSpec or not
+	Decorate bool `json:"decorate,omitempty"`
+
+	// PathAlias is the location under <root-dir>/src
+	// where the repository under test is cloned. If this
+	// is not set, <root-dir>/src/github.com/org/repo will
+	// be used as the default.
+	PathAlias string `json:"pathAlias,omitempty"`
+	// CloneURI is the URI that is used to clone the
+	// repository. If unset, will default to
+	// `https://github.com/org/repo.git`.
+	CloneURI string `json:"cloneUri,omitempty"`
+	// SkipSubmodules determines if submodules should be
+	// cloned when the job is run. Defaults to true.
+	SkipSubmodules bool `json:"skipSubmodules,omitempty"`
+	// CloneDepth is the depth of the clone that will be used.
+	// A depth of zero will do a full clone.
+	CloneDepth int `json:"cloneDepth,omitempty"`
+}
diff --git a/pkg/apis/lighthouse/v1alpha1/repoconfig_plugins_types.go b/pkg/apis/lighthouse/v1alpha1/repoconfig_plugins_types.go
new file mode 100644
index 00000000..906d86c5
--- /dev/null
+++ b/pkg/apis/lighthouse/v1alpha1/repoconfig_plugins_types.go
@@ -0,0 +1,27 @@
+package v1alpha1
+
+// this file contains copies of structs from the plugins package
+// using different JSON/YAML marshalling so that they look like
+// kubernetes CRDs; using camelCase rather than lower case and underscores
+
+// Approve specifies a configuration for a single approve.
+//
+// The configuration for the approve plugin is defined as a list of these structures.
+type Approve struct {
+	// IssueRequired indicates if an associated issue is required for approval in
+	// the specified repos.
+	IssueRequired bool `json:"issueRequired,omitempty"`
+
+	// RequireSelfApproval requires PR authors to explicitly approve their PRs.
+	// Otherwise the plugin assumes the author of the PR approves the changes in the PR.
+	RequireSelfApproval *bool `json:"requireSelfApproval,omitempty"`
+
+	// LgtmActsAsApprove indicates that the lgtm command should be used to
+	// indicate approval
+	LgtmActsAsApprove bool `json:"lgtmActsAsApprove,omitempty"`
+
+	// IgnoreReviewState causes the approve plugin to ignore the GitHub review state. Otherwise:
+	// * an APPROVE github review is equivalent to leaving an "/approve" message.
+	// * A REQUEST_CHANGES github review is equivalent to leaving an /approve cancel" message.
+	IgnoreReviewState *bool `json:"ignoreReviewState,omitempty"`
+}
diff --git a/pkg/apis/lighthouse/v1alpha1/repoconfig_types.go b/pkg/apis/lighthouse/v1alpha1/repoconfig_types.go
new file mode 100644
index 00000000..efa73d51
--- /dev/null
+++ b/pkg/apis/lighthouse/v1alpha1/repoconfig_types.go
@@ -0,0 +1,69 @@
+package v1alpha1
+
+import (
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +genclient
+// +genclient:noStatus
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// RepositoryConfig represents local repository configurations
+//
+// +k8s:openapi-gen=true
+type RepositoryConfig struct {
+	metav1.TypeMeta `json:",inline"`
+	// +optional
+	metav1.ObjectMeta `json:"metadata"`
+
+	// Spec holds the desired state of the RepositoryConfig from the client
+	// +optional
+	Spec RepositoryConfigSpec `json:"spec"`
+}
+
+// RepositoryConfigSpec defines the desired state of RepositoryConfig.
+type RepositoryConfigSpec struct {
+	// Presubmit zero or more presubmits
+	Presubmits []Presubmit `json:"presubmits,omitempty"`
+
+	// Postsubmit zero or more postsubmits
+	Postsubmits []Postsubmit `json:"postsubmits,omitempty"`
+
+	// Plugins the plugin names to enable for this repository
+	Plugins []string `json:"plugins,omitempty"`
+
+	// BranchProtections to configure branch protection
+	BranchProtection *config.ContextPolicy `json:"branchProtection,omitempty"`
+
+	// PluginConfig the configuration for the plugins
+	PluginConfig *PluginConfig `json:"pluginConfig,omitempty"`
+
+	// Keeper any keeper/tide related configurations
+	Keeper *Keeper `json:"keeper,omitempty"`
+}
+
+// RepositoryConfigList contains a list of RepositoryConfig
+//
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type RepositoryConfigList struct {
+	metav1.TypeMeta `json:",inline"`
+	// +optional
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []RepositoryConfig `json:"items"`
+}
+
+// Keeper for keeper specific queries
+type Keeper struct {
+	// Query the query to add for the repository
+	Query *config.KeeperQuery `json:"query,omitempty"`
+}
+
+// PluginConfig configuration for plugins
+type PluginConfig struct {
+	// Plugins the list of plugins
+	Plugins []string `json:"plugins,omitempty"`
+
+	// Approve approve plugin configuration
+	Approve *Approve `json:"approve,omitempty"`
+}
diff --git a/pkg/repoconfig/merge/combine.go b/pkg/repoconfig/merge/combine.go
new file mode 100644
index 00000000..a236fe47
--- /dev/null
+++ b/pkg/repoconfig/merge/combine.go
@@ -0,0 +1,31 @@
+package merge
+
+import (
+	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+)
+
+// CombineConfigs combines the two configurations together from multiple files
+func CombineConfigs(a, b *v1alpha1.RepositoryConfig) *v1alpha1.RepositoryConfig {
+	if a == nil {
+		return b
+	}
+	if b == nil {
+		return a
+	}
+	for _, r := range b.Spec.Presubmits {
+		a.Spec.Presubmits = append(a.Spec.Presubmits, r)
+	}
+	for _, r := range b.Spec.Postsubmits {
+		a.Spec.Postsubmits = append(a.Spec.Postsubmits, r)
+	}
+	if b.Spec.BranchProtection != nil {
+		if a.Spec.BranchProtection == nil {
+			a.Spec.BranchProtection = &config.ContextPolicy{}
+		}
+		for _, s := range b.Spec.BranchProtection.Contexts {
+			a.Spec.BranchProtection.Contexts = append(a.Spec.BranchProtection.Contexts, s)
+		}
+	}
+	return a
+}
diff --git a/pkg/repoconfig/merge/converters.go b/pkg/repoconfig/merge/converters.go
new file mode 100644
index 00000000..d11f9917
--- /dev/null
+++ b/pkg/repoconfig/merge/converters.go
@@ -0,0 +1,92 @@
+package merge
+
+import (
+	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+)
+
+// ToPresubmit converts the repo config to the lighthouse config resource
+func ToPresubmit(r v1alpha1.Presubmit) config.Presubmit {
+	return config.Presubmit{
+		JobBase:             ToJobBase(r.JobBase),
+		AlwaysRun:           r.AlwaysRun,
+		Optional:            r.Optional,
+		Trigger:             r.Trigger,
+		RerunCommand:        r.RerunCommand,
+		Brancher:            ToBrancher(r.Brancher),
+		RegexpChangeMatcher: ToRegexpChangeMatcher(r.RegexpChangeMatcher),
+		Reporter:            ToReporter(r.Reporter),
+	}
+}
+
+// ToPostsubmit converts the repo config to the lighthouse config resource
+func ToPostsubmit(r v1alpha1.Postsubmit) config.Postsubmit {
+	return config.Postsubmit{
+		JobBase:             ToJobBase(r.JobBase),
+		RegexpChangeMatcher: ToRegexpChangeMatcher(r.RegexpChangeMatcher),
+		Brancher:            ToBrancher(r.Brancher),
+		Reporter:            ToReporter(r.Reporter),
+	}
+}
+
+// ToJobBase converts the repo config to the lighthouse config resource
+func ToJobBase(r v1alpha1.JobBase) config.JobBase {
+	return config.JobBase{
+		Name:            r.Name,
+		Labels:          r.Labels,
+		Annotations:     r.Annotations,
+		MaxConcurrency:  r.MaxConcurrency,
+		Agent:           r.Agent,
+		Cluster:         r.Cluster,
+		Namespace:       r.Namespace,
+		ErrorOnEviction: r.ErrorOnEviction,
+		SourcePath:      r.SourcePath,
+		Spec:            r.Spec,
+		PipelineRunSpec: r.PipelineRunSpec,
+		UtilityConfig:   ToUtilityConfig(r.UtilityConfig),
+	}
+}
+
+// ToBrancher converts the repo config to the lighthouse config resource
+func ToBrancher(r v1alpha1.Brancher) config.Brancher {
+	return config.Brancher{
+		SkipBranches: r.SkipBranches,
+		Branches:     r.Branches,
+	}
+}
+
+// ToReporter converts the repo config to the lighthouse config resource
+func ToReporter(r v1alpha1.Reporter) config.Reporter {
+	return config.Reporter{
+		Context:    r.Context,
+		SkipReport: r.SkipReport,
+	}
+}
+
+// ToRegexpChangeMatcher converts the repo config to the lighthouse config resource
+func ToRegexpChangeMatcher(r v1alpha1.RegexpChangeMatcher) config.RegexpChangeMatcher {
+	return config.RegexpChangeMatcher{RunIfChanged: r.RunIfChanged}
+}
+
+// ToUtilityConfig converts the repo config to the lighthouse config resource
+func ToUtilityConfig(r v1alpha1.UtilityConfig) config.UtilityConfig {
+	return config.UtilityConfig{
+		Decorate:       r.Decorate,
+		PathAlias:      r.PathAlias,
+		CloneURI:       r.CloneURI,
+		SkipSubmodules: r.SkipSubmodules,
+		CloneDepth:     r.CloneDepth,
+	}
+}
+
+// ToApprove converts the repo config to the lighthouse config resource
+func ToApprove(r *v1alpha1.Approve, repoKey string) plugins.Approve {
+	return plugins.Approve{
+		Repos:               []string{repoKey},
+		IssueRequired:       r.IssueRequired,
+		RequireSelfApproval: r.RequireSelfApproval,
+		LgtmActsAsApprove:   r.LgtmActsAsApprove,
+		IgnoreReviewState:   r.IgnoreReviewState,
+	}
+}
diff --git a/pkg/repoconfig/merge/helpers.go b/pkg/repoconfig/merge/helpers.go
new file mode 100644
index 00000000..d8e9cd43
--- /dev/null
+++ b/pkg/repoconfig/merge/helpers.go
@@ -0,0 +1,18 @@
+package merge
+
+// StringArrayIndex returns the index in the slice which equals the given value
+func StringArrayIndex(array []string, value string) int {
+	for i, v := range array {
+		if v == value {
+			return i
+		}
+	}
+	return -1
+}
+
+// RemoveStringArrayAtIndex removes an item at a given index
+func RemoveStringArrayAtIndex(s []string, index int) []string {
+	ret := make([]string, 0)
+	ret = append(ret, s[:index]...)
+	return append(ret, s[index+1:]...)
+}
diff --git a/pkg/repoconfig/merge/merge.go b/pkg/repoconfig/merge/merge.go
new file mode 100644
index 00000000..d4d712a0
--- /dev/null
+++ b/pkg/repoconfig/merge/merge.go
@@ -0,0 +1,88 @@
+package merge
+
+import (
+	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+)
+
+// ConfigMerge merges the repository configuration into the global configuration
+func ConfigMerge(cfg *config.Config, pluginsCfg *plugins.Configuration, repoConfig *v1alpha1.RepositoryConfig, repoOwner string, repoName string) error {
+	repoKey := repoOwner + "/" + repoName
+	if len(repoConfig.Spec.Presubmits) > 0 {
+		if cfg.Presubmits == nil {
+			cfg.Presubmits = map[string][]config.Presubmit{}
+		}
+
+		var ps []config.Presubmit
+		for _, p := range repoConfig.Spec.Presubmits {
+			ps = append(ps, ToPresubmit(p))
+		}
+		cfg.Presubmits[repoKey] = ps
+	}
+	if len(repoConfig.Spec.Postsubmits) > 0 {
+		if cfg.Postsubmits == nil {
+			cfg.Postsubmits = map[string][]config.Postsubmit{}
+		}
+
+		var ps []config.Postsubmit
+		for _, p := range repoConfig.Spec.Postsubmits {
+			ps = append(ps, ToPostsubmit(p))
+		}
+		cfg.Postsubmits[repoKey] = ps
+	}
+
+	if repoConfig.Spec.BranchProtection != nil {
+		if cfg.BranchProtection.Orgs == nil {
+			cfg.BranchProtection.Orgs = map[string]config.Org{}
+		}
+		org := cfg.BranchProtection.Orgs[repoOwner]
+		if org.Repos == nil {
+			org.Repos = map[string]config.Repo{}
+		}
+		repo := org.Repos[repoName]
+		repo.RequiredStatusChecks = repoConfig.Spec.BranchProtection
+		org.Repos[repoName] = repo
+		cfg.BranchProtection.Orgs[repoOwner] = org
+	}
+
+	keeper := repoConfig.Spec.Keeper
+	if keeper != nil {
+		if keeper.Query != nil {
+			for i, q := range cfg.Keeper.Queries {
+				// remove the old repository
+				idx := StringArrayIndex(q.Repos, repoKey)
+				if idx >= 0 {
+					cfg.Keeper.Queries[i].Repos = RemoveStringArrayAtIndex(cfg.Keeper.Queries[i].Repos, idx)
+					break
+				}
+			}
+			query := *keeper.Query
+			query.Repos = []string{repoKey}
+			cfg.Keeper.Queries = append(cfg.Keeper.Queries, query)
+		}
+	}
+
+	pc := repoConfig.Spec.PluginConfig
+	if pc != nil {
+		if pluginsCfg.Plugins == nil {
+			pluginsCfg.Plugins = map[string][]string{}
+		}
+		pluginsCfg.Plugins[repoKey] = pc.Plugins
+
+		if pc.Approve != nil {
+			pluginsCfg.Approve = append(pluginsCfg.Approve, ToApprove(pc.Approve, repoKey))
+		}
+
+		// lets add to the last idx
+		idx := len(pluginsCfg.Triggers) - 1
+		if idx < 0 {
+			idx = 0
+			pluginsCfg.Triggers = append(pluginsCfg.Triggers, plugins.Trigger{})
+		}
+		if StringArrayIndex(pluginsCfg.Triggers[idx].Repos, repoKey) < 0 {
+			pluginsCfg.Triggers[idx].Repos = append(pluginsCfg.Triggers[idx].Repos, repoKey)
+		}
+	}
+	return nil
+}
diff --git a/pkg/repoconfig/scmload/load.go b/pkg/repoconfig/scmload/load.go
new file mode 100644
index 00000000..6285665f
--- /dev/null
+++ b/pkg/repoconfig/scmload/load.go
@@ -0,0 +1,93 @@
+package scmload
+
+import (
+	"context"
+	"strings"
+
+	"github.com/jenkins-x/go-scm/scm"
+	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+	"github.com/jenkins-x/lighthouse/pkg/repoconfig/merge"
+	"github.com/pkg/errors"
+	"sigs.k8s.io/yaml"
+)
+
+// MergeConfig merges the configuration with any `lighthouse.yaml` files in the repository
+func MergeConfig(cfg *config.Config, pluginCfg *plugins.Configuration, scmClient *scm.Client, repoOwner string, repoName string, sha string) (bool, error) {
+	repoConfig, err := LoadRepositoryConfig(scmClient, repoOwner, repoName, sha)
+	if err != nil {
+		return false, errors.Wrap(err, "failed to load configs")
+	}
+	if repoConfig == nil {
+		return false, nil
+	}
+
+	err = merge.ConfigMerge(cfg, pluginCfg, repoConfig, repoOwner, repoName)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to merge repository config with repository %s/%s and sha %s", repoOwner, repoName, sha)
+	}
+	return true, nil
+}
+
+// LoadRepositoryConfig loads the `lighthouse.yaml` configuration files in the repository
+func LoadRepositoryConfig(scmClient *scm.Client, repoOwner string, repoName string, sha string) (*v1alpha1.RepositoryConfig, error) {
+	if sha == "" {
+		sha = "master"
+	}
+
+	var answer *v1alpha1.RepositoryConfig
+	repo := scm.Join(repoOwner, repoName)
+
+	ctx := context.Background()
+	path := ".lighthouse"
+	files, _, err := scmClient.Contents.List(ctx, repo, path, sha)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find any lighthouse configuration files in repo %s at sha %s", repo, sha)
+	}
+	for _, f := range files {
+		if isDirType(f.Type) {
+			filePath := path + "/" + f.Name + "/lighthouse.yaml"
+			cfg, err := loadConfigFile(ctx, scmClient, repo, filePath, sha)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to load file %s in %s with sha %s", filePath, repo, sha)
+			}
+			if cfg != nil {
+				answer = merge.CombineConfigs(answer, cfg)
+			}
+		} else if f.Name == "lighthouse.yaml" {
+			filePath := path + "/" + f.Name
+			cfg, err := loadConfigFile(ctx, scmClient, repo, filePath, sha)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to load file %s in %s with sha %s", filePath, repo, sha)
+			}
+			if cfg != nil {
+				answer = merge.CombineConfigs(answer, cfg)
+			}
+		}
+	}
+	return answer, nil
+}
+
+func isDirType(t string) bool {
+	return strings.ToLower(t) == "dir"
+}
+
+func loadConfigFile(ctx context.Context, client *scm.Client, repo string, path string, sha string) (*v1alpha1.RepositoryConfig, error) {
+	c, r, err := client.Contents.Find(ctx, repo, path, sha)
+	if err != nil {
+		if r != nil && r.Status == 404 {
+			return nil, nil
+		}
+		return nil, errors.Wrapf(err, "failed to find  file %s in repo %s with sha %s status %d", path, repo, sha, r.Status)
+	}
+	if len(c.Data) == 0 {
+		return nil, nil
+	}
+	repoConfig := &v1alpha1.RepositoryConfig{}
+	err = yaml.Unmarshal(c.Data, repoConfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to unmarshal file %s in repo %s with sha %s", path, repo, sha)
+	}
+	return repoConfig, nil
+}


Test patch:
diff --git a/pkg/repoconfig/merge/combine_test.go b/pkg/repoconfig/merge/combine_test.go
new file mode 100644
index 00000000..94f8b7e9
--- /dev/null
+++ b/pkg/repoconfig/merge/combine_test.go
@@ -0,0 +1,104 @@
+package merge_test
+
+import (
+	"testing"
+
+	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
+	"github.com/jenkins-x/lighthouse/pkg/repoconfig/merge"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCombineRepositoryConfig(t *testing.T) {
+	v1 := v1alpha1.RepositoryConfig{
+		Spec: v1alpha1.RepositoryConfigSpec{
+			Presubmits: []v1alpha1.Presubmit{
+				{
+					JobBase: v1alpha1.JobBase{
+						Name: "lint",
+					},
+					AlwaysRun:    true,
+					Optional:     false,
+					Trigger:      "/lint",
+					RerunCommand: "/relint",
+					Reporter: v1alpha1.Reporter{
+						Context: "lint",
+					},
+				},
+			},
+			Postsubmits: []v1alpha1.Postsubmit{
+				{
+					JobBase: v1alpha1.JobBase{
+						Name: "release",
+					},
+					Reporter: v1alpha1.Reporter{
+						Context: "release",
+					},
+				},
+			},
+		},
+	}
+	v2 := v1alpha1.RepositoryConfig{
+		Spec: v1alpha1.RepositoryConfigSpec{
+			Presubmits: []v1alpha1.Presubmit{
+				{
+					JobBase: v1alpha1.JobBase{
+						Name: "another",
+					},
+					AlwaysRun:    true,
+					Optional:     false,
+					Trigger:      "/another",
+					RerunCommand: "/reanother",
+					Reporter: v1alpha1.Reporter{
+						Context: "another",
+					},
+				},
+			},
+		},
+	}
+
+	testCases := []struct {
+		name                string
+		r1                  *v1alpha1.RepositoryConfig
+		r2                  *v1alpha1.RepositoryConfig
+		expectedPresubmits  int
+		expectedPostsubmits int
+	}{
+		{
+			name: "bothNil",
+		},
+		{
+			name:                "r1",
+			r1:                  &v1,
+			expectedPostsubmits: 1,
+			expectedPresubmits:  1,
+		},
+		{
+			name:                "r2",
+			r2:                  &v1,
+			expectedPostsubmits: 1,
+			expectedPresubmits:  1,
+		},
+		{
+			name:                "combine",
+			r1:                  &v1,
+			r2:                  &v2,
+			expectedPostsubmits: 1,
+			expectedPresubmits:  2,
+		},
+	}
+
+	for _, tc := range testCases {
+		name := tc.name
+		actual := merge.CombineConfigs(tc.r1, tc.r2)
+
+		if tc.r1 == nil && tc.r2 == nil {
+			assert.Nil(t, actual, "expectedPresubmits nil results for %s", name)
+		} else {
+			require.NotNil(t, actual, "nil results for %s", name)
+
+			assert.Len(t, actual.Spec.Presubmits, tc.expectedPresubmits, "expected presubmits for %s", name)
+			assert.Len(t, actual.Spec.Postsubmits, tc.expectedPostsubmits, "expected postsubmits for %s", name)
+		}
+	}
+}
diff --git a/pkg/repoconfig/merge/merge_test.go b/pkg/repoconfig/merge/merge_test.go
new file mode 100644
index 00000000..55687707
--- /dev/null
+++ b/pkg/repoconfig/merge/merge_test.go
@@ -0,0 +1,154 @@
+package merge_test
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+	"github.com/jenkins-x/lighthouse/pkg/repoconfig/merge"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/yaml"
+)
+
+func TestMergeRepositoryConfig(t *testing.T) {
+	testCases := []struct {
+		name       string
+		cfg        config.Config
+		pluginCfg  plugins.Configuration
+		repoConfig v1alpha1.RepositoryConfig
+	}{
+		{
+			name: "emptyConfig",
+			repoConfig: v1alpha1.RepositoryConfig{
+				Spec: v1alpha1.RepositoryConfigSpec{
+					Presubmits: []v1alpha1.Presubmit{
+						{
+							JobBase: v1alpha1.JobBase{
+								Name: "lint",
+							},
+							AlwaysRun:    true,
+							Optional:     false,
+							Trigger:      "/lint",
+							RerunCommand: "/relint",
+							Reporter: v1alpha1.Reporter{
+								Context: "lint",
+							},
+						},
+					},
+					Postsubmits: []v1alpha1.Postsubmit{
+						{
+							JobBase: v1alpha1.JobBase{
+								Name: "release",
+							},
+							Reporter: v1alpha1.Reporter{
+								Context: "release",
+							},
+						},
+					},
+					Keeper: &v1alpha1.Keeper{
+						Query: &config.KeeperQuery{
+							Labels:        []string{"mylabel"},
+							MissingLabels: []string{"foo", "bar"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	repoOwner := "myorg"
+	repoName := "myowner"
+	repoKey := repoOwner + "/" + repoName
+
+	for _, tc := range testCases {
+		name := tc.name
+		err := merge.ConfigMerge(&tc.cfg, &tc.pluginCfg, &tc.repoConfig, repoOwner, repoName)
+		require.NoError(t, err, "failed to merge repository config for %s", name)
+
+		assert.Equal(t, len(tc.repoConfig.Spec.Presubmits), len(tc.cfg.Presubmits[repoKey]), "presubmits for %s", name)
+		t.Logf("test %s has %d presubmits for repository key %s", name, len(tc.cfg.Presubmits[repoKey]), repoKey)
+
+		assert.Equal(t, len(tc.repoConfig.Spec.Postsubmits), len(tc.cfg.Postsubmits[repoKey]), "postsubmits for %s", name)
+		t.Logf("test %s has %d postsubmits for repository key %s", name, len(tc.cfg.Postsubmits[repoKey]), repoKey)
+	}
+}
+
+func TestMergeRepositoryConfigFiles(t *testing.T) {
+	sourceData := filepath.Join("test_data")
+	fileNames, err := ioutil.ReadDir(sourceData)
+	assert.NoError(t, err)
+
+	repoOwner := "myorg"
+	repoName := "myowner"
+
+	for _, f := range fileNames {
+		if f.IsDir() {
+			name := f.Name()
+			srcConfigFile := filepath.Join(sourceData, name, "source-config.yaml")
+			srcPluginsFile := filepath.Join(sourceData, name, "source-plugins.yaml")
+			expectedConfigFile := filepath.Join(sourceData, name, "expected-config.yaml")
+			expectedPluginsFile := filepath.Join(sourceData, name, "expected-plugins.yaml")
+			repoConfigFile := filepath.Join(sourceData, name, "lighthouse.yaml")
+			require.FileExists(t, srcConfigFile)
+			require.FileExists(t, expectedConfigFile)
+			require.FileExists(t, expectedPluginsFile)
+			require.FileExists(t, repoConfigFile)
+
+			cfg := &config.Config{}
+			pluginCfg := &plugins.Configuration{}
+			repoConfig := &v1alpha1.RepositoryConfig{}
+			LoadYAMLFile(t, srcConfigFile, cfg)
+			LoadYAMLFile(t, srcPluginsFile, pluginCfg)
+			LoadYAMLFile(t, repoConfigFile, repoConfig)
+
+			err := merge.ConfigMerge(cfg, pluginCfg, repoConfig, repoOwner, repoName)
+			require.NoError(t, err, "failed to merge files in dir %s", name)
+
+			resultConfigText := ToYAMLString(t, cfg, name)
+			resultPluginsText := ToYAMLString(t, pluginCfg, name)
+
+			expectedConfigText := LoadTrimmedText(t, expectedConfigFile)
+			expectedPluginsText := LoadTrimmedText(t, expectedPluginsFile)
+
+			if d := cmp.Diff(strings.TrimSpace(resultConfigText), expectedConfigText); d != "" {
+				t.Errorf("Generated config did not match expected: %s", d)
+			}
+			if d := cmp.Diff(strings.TrimSpace(resultPluginsText), expectedPluginsText); d != "" {
+				t.Errorf("Generated plugins did not match expected: %s", d)
+			}
+			t.Logf("generated for file %s\n%s\n", srcConfigFile, resultConfigText)
+			t.Logf("generated for file %s\n%s\n", srcPluginsFile, resultPluginsText)
+		}
+	}
+}
+
+func LoadTrimmedText(t *testing.T, expectedConfigFile string) string {
+	expectData, err := ioutil.ReadFile(expectedConfigFile)
+	require.NoError(t, err, "failed to load results %s", expectedConfigFile)
+	expectedText := strings.TrimSpace(string(expectData))
+	return expectedText
+}
+
+func ToYAMLString(t *testing.T, cfg interface{}, testName string) string {
+	resultData, err := yaml.Marshal(&cfg)
+	require.NoError(t, err, "failed to marshal config for %s", testName)
+	resultText := string(resultData)
+	return resultText
+}
+
+// LoadFile loads the given YAML file
+func LoadYAMLFile(t *testing.T, fileName string, dest interface{}) {
+	require.FileExists(t, fileName)
+
+	data, err := ioutil.ReadFile(fileName)
+	require.NoError(t, err, "failed to read file %s", fileName)
+
+	err = yaml.Unmarshal(data, dest)
+	require.NoError(t, err, "failed to unmarshal YAML file %s", fileName)
+}
diff --git a/pkg/repoconfig/merge/test_data/simple/expected-config.yaml b/pkg/repoconfig/merge/test_data/simple/expected-config.yaml
new file mode 100644
index 00000000..1d463cbc
--- /dev/null
+++ b/pkg/repoconfig/merge/test_data/simple/expected-config.yaml
@@ -0,0 +1,44 @@
+branch-protection:
+  orgs:
+    myorg:
+      repos:
+        myowner:
+          required_status_checks:
+            contexts:
+            - lint
+deck:
+  spyglass: {}
+gerrit: {}
+github:
+  LinkURL: null
+owners_dir_blacklist:
+  default: null
+  repos: null
+plank: {}
+postsubmits:
+  myorg/myowner:
+  - agent: ""
+    context: release
+    name: release
+presubmits:
+  myorg/myowner:
+  - agent: ""
+    always_run: true
+    context: lint
+    name: lint
+    rerun_command: /relint
+    trigger: /lint
+push_gateway:
+  serve_metrics: false
+sinker: {}
+tide:
+  context_options:
+    required-if-present-contexts: null
+  queries:
+  - labels:
+    - mylabel
+    missingLabels:
+    - foo
+    - bar
+    repos:
+    - myorg/myowner
\ No newline at end of file
diff --git a/pkg/repoconfig/merge/test_data/simple/expected-plugins.yaml b/pkg/repoconfig/merge/test_data/simple/expected-plugins.yaml
new file mode 100644
index 00000000..56a0e168
--- /dev/null
+++ b/pkg/repoconfig/merge/test_data/simple/expected-plugins.yaml
@@ -0,0 +1,65 @@
+approve:
+- lgtm_acts_as_approve: true
+  repos:
+  - someorg/somerepo
+  require_self_approval: true
+- repos:
+  - myorg/myowner
+  require_self_approval: false
+blunderbuss: {}
+cat: {}
+cherry_pick_unapproved: {}
+config_updater:
+  gzip: false
+  maps:
+    env/prow/config.yaml:
+      name: config
+    env/prow/plugins.yaml:
+      name: plugins
+heart: {}
+label:
+  additional_labels: null
+owners: {}
+plugins:
+  myorg/myowner:
+  - approve
+  - assign
+  - help
+  - hold
+  - lgtm
+  - lifecycle
+  - size
+  - trigger
+  - wip
+  - heart
+  someorg/somerepo:
+  - config-updater
+  - approve
+  - assign
+  - blunderbuss
+  - help
+  - hold
+  - lgtm
+  - lifecycle
+  - size
+  - trigger
+  - wip
+  - heart
+  - cat
+  - override
+  - dog
+  - pony
+requiresig: {}
+sigmention: {}
+size:
+  l: 0
+  m: 0
+  s: 0
+  xl: 0
+  xxl: 0
+slack: {}
+triggers:
+- repos:
+  - someorg/somerepo
+  - myorg/myowner
+  trusted_org: someorg
\ No newline at end of file
diff --git a/pkg/repoconfig/merge/test_data/simple/lighthouse.yaml b/pkg/repoconfig/merge/test_data/simple/lighthouse.yaml
new file mode 100644
index 00000000..7815324a
--- /dev/null
+++ b/pkg/repoconfig/merge/test_data/simple/lighthouse.yaml
@@ -0,0 +1,38 @@
+apiVersion: config.lighthouse.jenkins-x.io/v1alpha1
+kind: RepositoryConfig
+spec:
+  presubmits:
+  - name: lint
+    context: "lint"
+    alwaysRun: true
+    optional: false
+    trigger: "/lint"
+    rerunCommand: "/relint"
+  postsubmits:
+  - name: release
+    context: "release"
+  branchProtection:
+    contexts:
+    - lint
+  pluginConfig:
+    plugins:
+    - approve
+    - assign
+    - help
+    - hold
+    - lgtm
+    - lifecycle
+    - size
+    - trigger
+    - wip
+    - heart
+    approve:
+      lgtmActsAsApprove: false
+      requireSelfApproval: false
+  keeper:
+    query:
+      labels:
+      - mylabel
+      missingLabels:
+      - foo
+      - bar
diff --git a/pkg/repoconfig/merge/test_data/simple/source-config.yaml b/pkg/repoconfig/merge/test_data/simple/source-config.yaml
new file mode 100644
index 00000000..a734d431
--- /dev/null
+++ b/pkg/repoconfig/merge/test_data/simple/source-config.yaml
@@ -0,0 +1,17 @@
+branch-protection: {}
+deck:
+  spyglass: {}
+gerrit: {}
+github:
+  LinkURL: null
+owners_dir_blacklist:
+  default: null
+  repos: null
+plank: {}
+presubmits: {}
+push_gateway:
+  serve_metrics: false
+sinker: {}
+tide:
+  context_options:
+    required-if-present-contexts: null
\ No newline at end of file
diff --git a/pkg/repoconfig/merge/test_data/simple/source-plugins.yaml b/pkg/repoconfig/merge/test_data/simple/source-plugins.yaml
new file mode 100644
index 00000000..e7f3e715
--- /dev/null
+++ b/pkg/repoconfig/merge/test_data/simple/source-plugins.yaml
@@ -0,0 +1,49 @@
+approve:
+- lgtm_acts_as_approve: true
+  repos:
+  - someorg/somerepo
+  require_self_approval: true
+blunderbuss: {}
+cat: {}
+cherry_pick_unapproved: {}
+config_updater:
+  maps:
+    env/prow/config.yaml:
+      name: config
+    env/prow/plugins.yaml:
+      name: plugins
+heart: {}
+label:
+  additional_labels: null
+owners: {}
+plugins:
+  someorg/somerepo:
+  - config-updater
+  - approve
+  - assign
+  - blunderbuss
+  - help
+  - hold
+  - lgtm
+  - lifecycle
+  - size
+  - trigger
+  - wip
+  - heart
+  - cat
+  - override
+  - dog
+  - pony
+requiresig: {}
+sigmention: {}
+size:
+  l: 0
+  m: 0
+  s: 0
+  xl: 0
+  xxl: 0
+slack: {}
+triggers:
+- repos:
+  - someorg/somerepo
+  trusted_org: someorg
\ No newline at end of file
diff --git a/pkg/repoconfig/scmload/load_integration_test.go b/pkg/repoconfig/scmload/load_integration_test.go
new file mode 100644
index 00000000..7b3b14b3
--- /dev/null
+++ b/pkg/repoconfig/scmload/load_integration_test.go
@@ -0,0 +1,57 @@
+package scmload_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/jenkins-x/go-scm/scm/factory"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+	"github.com/jenkins-x/lighthouse/pkg/repoconfig/scmload"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeConfigIntegration(t *testing.T) {
+	token := os.Getenv("GIT_TOKEN")
+	if token == "" {
+		t.Logf("no $GIT_TOKEN defined so skipping this integration test")
+		t.SkipNow()
+		return
+	}
+	scmClient, _ := factory.NewClient("github", "https://github.com", token)
+	repoOwner := "jenkins-x"
+	repoName := "lighthouse-test-project"
+	sha := ""
+	cfg := &config.Config{}
+	pluginCfg := &plugins.Configuration{}
+
+	flag, err := scmload.MergeConfig(cfg, pluginCfg, scmClient, repoOwner, repoName, sha)
+	require.NoError(t, err, "failed to merge configs")
+	assert.True(t, flag, "did not return merge flag")
+
+	// lets display the context
+	LogConfig(t, cfg)
+
+	r := repoOwner + "/" + repoName
+	assert.Len(t, cfg.Presubmits[r], 2, "presubmits for repo %s", r)
+	assert.Len(t, cfg.Postsubmits[r], 1, "postsubmits for repo %s", r)
+}
+
+// LogConfig displays the generated config
+func LogConfig(t *testing.T, cfg *config.Config) {
+	for k, v := range cfg.Presubmits {
+		t.Logf("presubmits for repository %s\n", k)
+
+		for _, r := range v {
+			t.Logf("  presubmit %s\n", r.Name)
+		}
+	}
+	for k, v := range cfg.Postsubmits {
+		t.Logf("postsubmits for repository %s\n", k)
+
+		for _, r := range v {
+			t.Logf("  postsubmits %s\n", r.Name)
+		}
+	}
+}
diff --git a/pkg/repoconfig/scmload/load_test.go b/pkg/repoconfig/scmload/load_test.go
new file mode 100644
index 00000000..c3c82213
--- /dev/null
+++ b/pkg/repoconfig/scmload/load_test.go
@@ -0,0 +1,30 @@
+package scmload_test
+
+import (
+	"testing"
+
+	"github.com/jenkins-x/go-scm/scm/driver/fake"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+	"github.com/jenkins-x/lighthouse/pkg/repoconfig/scmload"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeConfig(t *testing.T) {
+	scmClient, _ := fake.NewDefault()
+	repoOwner := "myorg"
+	repoName := "myrepo"
+	sha := "TODO"
+	cfg := &config.Config{}
+	pluginCfg := &plugins.Configuration{}
+	flag, err := scmload.MergeConfig(cfg, pluginCfg, scmClient, repoOwner, repoName, sha)
+	require.NoError(t, err, "failed to merge configs")
+	assert.True(t, flag, "did not return merge flag")
+
+	LogConfig(t, cfg)
+
+	r := repoOwner + "/" + repoName
+	assert.Len(t, cfg.Presubmits[r], 2, "presubmits for repo %s", r)
+	assert.Len(t, cfg.Postsubmits[r], 1, "postsubmits for repo %s", r)
+}
diff --git a/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/dummy/README.md b/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/dummy/README.md
new file mode 100644
index 00000000..8957ece3
--- /dev/null
+++ b/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/dummy/README.md
@@ -0,0 +1,1 @@
+to test a dir without a config file
diff --git a/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/jenkins-x/lighthouse.yaml b/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/jenkins-x/lighthouse.yaml
new file mode 100644
index 00000000..9af5e1d1
--- /dev/null
+++ b/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/jenkins-x/lighthouse.yaml
@@ -0,0 +1,17 @@
+apiVersion: config.lighthouse.jenkins-x.io/v1alpha1
+kind: RepositoryConfig
+spec:
+  presubmits:
+  - name: test
+    context: "test"
+    alwaysRun: true
+    optional: false
+    trigger: "/test"
+    rerunCommand: "/retest"
+  postsubmits:
+  - name: release
+    context: "release"
+  branchProtection:
+    contexts:
+    - test
+
diff --git a/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/linter/lighthouse.yaml b/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/linter/lighthouse.yaml
new file mode 100644
index 00000000..fb8f4c3d
--- /dev/null
+++ b/pkg/repoconfig/scmload/test_data/myorg/myrepo/.lighthouse/linter/lighthouse.yaml
@@ -0,0 +1,14 @@
+apiVersion: config.lighthouse.jenkins-x.io/v1alpha1
+kind: RepositoryConfig
+spec:
+  presubmits:
+  - name: lint
+    context: "lint"
+    alwaysRun: true
+    optional: false
+    trigger: "/lint"
+    rerunCommand: "/relint"
+  branchProtection:
+    contexts:
+    - lint
+