Repository: jenkins-x/lighthouse
Pull request #1340 (2021-07-02 11:47:46)

fix: remove the old go-scm based file browser
as it uses up too many API requests to use in the wild. This should
   simplify the code and make it easier to refactor going forward


Patch:
diff --git a/pkg/config/job/base.go b/pkg/config/job/base.go
index 1e57af2f..b8435391 100644
--- a/pkg/config/job/base.go
+++ b/pkg/config/job/base.go
@@ -22,6 +22,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/sirupsen/logrus"
+
 	tektonv1beta1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -68,6 +70,25 @@ type Base struct {
 	PipelineRunSpec *tektonv1beta1.PipelineRunSpec `json:"pipeline_run_spec,omitempty"`
 	// PipelineRunParams are the params used by the pipeline run
 	PipelineRunParams []PipelineRunParam `json:"pipeline_run_params,omitempty"`
+	// lets us register a loader
+	pipelineLoader func(*Base) error
+}
+
+// LoadPipeline() loads the pipeline specification if its not already been loaded
+func (b *Base) LoadPipeline(logger *logrus.Entry) error {
+	if b.PipelineRunSpec != nil || b.pipelineLoader == nil {
+		return nil
+	}
+	logger.Infof("lazy loading the PipelineRunSpec")
+	answer := b.pipelineLoader(b)
+	// lets gc the function
+	b.pipelineLoader = nil
+	return answer
+}
+
+// SetPipelineLoader sets the function to lazy load the pipeline spec
+func (b *Base) SetPipelineLoader(fn func(b *Base) error) {
+	b.pipelineLoader = fn
 }
 
 // SetDefaults initializes default values
diff --git a/pkg/externalplugincfg/load.go b/pkg/externalplugincfg/load.go
index 9e0dde44..abc1ea2d 100644
--- a/pkg/externalplugincfg/load.go
+++ b/pkg/externalplugincfg/load.go
@@ -2,6 +2,8 @@ package externalplugincfg
 
 import (
 	"context"
+	"strings"
+
 	"github.com/jenkins-x/lighthouse/pkg/util"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -9,7 +11,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/yaml"
-	"strings"
 )
 
 func LoadDisabledPlugins(l *logrus.Entry, client kubernetes.Interface, namespace string) ([]string, error) {
diff --git a/pkg/filebrowser/fake/fake_file_browser.go b/pkg/filebrowser/fake/fake_file_browser.go
index 6a40f1d5..e686c303 100644
--- a/pkg/filebrowser/fake/fake_file_browser.go
+++ b/pkg/filebrowser/fake/fake_file_browser.go
@@ -2,6 +2,7 @@ package fake
 
 import (
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/jenkins-x/lighthouse/pkg/util"
@@ -13,13 +14,15 @@ import (
 
 type fakeFileBrowser struct {
 	dir                      string
+	multiRepo                bool
 	mainAndCurrentBranchRefs []string
 }
 
 // NewFakeFileBrowser a simple fake provider pointing at a folder
-func NewFakeFileBrowser(dir string) filebrowser.Interface {
+func NewFakeFileBrowser(dir string, multiRepo bool) filebrowser.Interface {
 	return &fakeFileBrowser{
 		dir:                      dir,
+		multiRepo:                multiRepo,
 		mainAndCurrentBranchRefs: []string{"main"},
 	}
 }
@@ -29,13 +32,44 @@ func (f *fakeFileBrowser) GetMainAndCurrentBranchRefs(owner, repo, ref string) (
 }
 
 func (f *fakeFileBrowser) GetFile(owner, repo, path, ref string, fc filebrowser.FetchCache) ([]byte, error) {
-	fileName := filepath.Join(f.dir, path)
+	fileName := f.getPath(owner, repo, path, ref)
+	exists, err := util.FileExists(fileName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to check if file exists %s", fileName)
+	}
+	if !exists {
+		return nil, nil
+	}
 	/* #nosec */
 	return ioutil.ReadFile(fileName)
 }
 
+func (f *fakeFileBrowser) getPath(owner, repo, path, ref string) string {
+	if f.multiRepo {
+		refs := []string{ref}
+		// lets handle main or master as the default branch name
+		if ref == "main" {
+			refs = append(refs, "master")
+		}
+		for _, r := range refs {
+			p := filepath.Join(f.dir, owner, repo, "refs", r, path)
+			_, err := os.Stat(p)
+			if err == nil {
+				return p
+			}
+		}
+		return filepath.Join(f.dir, owner, repo, path)
+	}
+	return filepath.Join(f.dir, path)
+}
+
+func (f *fakeFileBrowser) WithDir(owner, repo, ref string, fc filebrowser.FetchCache, fn func(dir string) error) error {
+	dir := f.getPath(owner, repo, "", ref)
+	return fn(dir)
+}
+
 func (f *fakeFileBrowser) ListFiles(owner, repo, path, ref string, fc filebrowser.FetchCache) ([]*scm.FileEntry, error) {
-	dir := filepath.Join(f.dir, path)
+	dir := f.getPath(owner, repo, path, ref)
 	exists, err := util.DirExists(dir)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to check dir exists %s", dir)
@@ -55,14 +89,14 @@ func (f *fakeFileBrowser) ListFiles(owner, repo, path, ref string, fc filebrowse
 		if f.IsDir() {
 			t = "dir"
 		}
-		path := filepath.Join(dir, name)
+		childPath := filepath.Join(path, name)
 		answer = append(answer, &scm.FileEntry{
 			Name: name,
-			Path: path,
+			Path: childPath,
 			Type: t,
 			Size: int(f.Size()),
 			Sha:  ref,
-			Link: "file://" + path,
+			Link: "file://" + childPath,
 		})
 	}
 	return answer, nil
diff --git a/pkg/filebrowser/git_file_browser.go b/pkg/filebrowser/git_file_browser.go
index 17df05c6..2152cd05 100644
--- a/pkg/filebrowser/git_file_browser.go
+++ b/pkg/filebrowser/git_file_browser.go
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/git/v2"
@@ -37,6 +38,13 @@ func NewFileBrowserFromGitClient(clientFactory git.ClientFactory) Interface {
 	}
 }
 
+func (f *gitFileBrowser) WithDir(owner, repo, ref string, fc FetchCache, fn func(dir string) error) error {
+	return f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
+		dir := repoClient.Directory()
+		return fn(dir)
+	})
+}
+
 func (f *gitFileBrowser) GetMainAndCurrentBranchRefs(_, _, eventRef string) ([]string, error) {
 	return []string{"", eventRef}, nil
 }
@@ -44,7 +52,14 @@ func (f *gitFileBrowser) GetMainAndCurrentBranchRefs(_, _, eventRef string) ([]s
 func (f *gitFileBrowser) GetFile(owner, repo, path, ref string, fc FetchCache) (answer []byte, err error) {
 	err = f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
 		f := repoPath(repoClient, path)
-		var err error
+		exists, err := util.FileExists(f)
+		if err != nil {
+			return errors.Wrapf(err, "failed to check if file exists %s", f)
+		}
+		if !exists {
+			answer = nil
+			return nil
+		}
 		answer, err = ioutil.ReadFile(f) // #nosec
 		return err
 	})
@@ -186,18 +201,13 @@ func (c *repoClientFacade) UseRef(ref string, fc FetchCache) error {
 			}).Info("not fetching ref as we already have it")
 		}
 	}
-	if shouldFetch {
-		logrus.StandardLogger().WithFields(map[string]interface{}{
-			"Name": c.fullName,
-			"Ref":  ref,
-			"File": "git_file_browser",
-		}).Info("fetching ref")
-	}
 
 	if c.ref == ref && !shouldFetch {
 		return nil
 	}
 
+	start := time.Now()
+
 	// lets switch to the main branch first before we go to a custom sha/ref
 	if c.ref != "" && c.ref != c.mainBranch {
 		err := c.repoClient.Checkout(c.mainBranch)
@@ -224,6 +234,14 @@ func (c *repoClientFacade) UseRef(ref string, fc FetchCache) error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to checkout repository %s ref %s", c.fullName, ref)
 	}
+
+	duration := time.Now().Sub(start)
+	logrus.StandardLogger().WithFields(map[string]interface{}{
+		"Name":     c.fullName,
+		"Ref":      ref,
+		"File":     "git_file_browser",
+		"Duration": duration.String(),
+	}).Info("fetched and checked out ref")
 	return nil
 }
 
diff --git a/pkg/filebrowser/interface.go b/pkg/filebrowser/interface.go
index 482d229e..46b3b21d 100644
--- a/pkg/filebrowser/interface.go
+++ b/pkg/filebrowser/interface.go
@@ -12,4 +12,7 @@ type Interface interface {
 
 	// ListFiles returns the file and directory entries in the given path in the repository with the given sha
 	ListFiles(owner, repo, path, ref string, fc FetchCache) ([]*scm.FileEntry, error)
+
+	// WithDir processes the given repository and reference at the given directory
+	WithDir(owner, repo, ref string, fc FetchCache, f func(dir string) error) error
 }
diff --git a/pkg/filebrowser/scm_file_browser.go b/pkg/filebrowser/scm_file_browser.go
deleted file mode 100644
index 62ea0c8f..00000000
--- a/pkg/filebrowser/scm_file_browser.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package filebrowser
-
-import (
-	"strings"
-
-	"github.com/jenkins-x/go-scm/scm"
-	"github.com/pkg/errors"
-)
-
-type scmFileBrowser struct {
-	scmClient scmProviderClient
-}
-
-// NewFileBrowserFromScmClient creates a new file browser from an Scm client
-func NewFileBrowserFromScmClient(scmClient scmProviderClient) Interface {
-	return &scmFileBrowser{scmClient: scmClient}
-}
-
-func (f *scmFileBrowser) GetMainAndCurrentBranchRefs(owner, repo, eventRef string) ([]string, error) {
-	fullName := scm.Join(owner, repo)
-	repository, err := f.scmClient.GetRepositoryByFullName(fullName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find repository %s", fullName)
-	}
-	mainBranch := repository.Branch
-	if mainBranch == "" {
-		mainBranch = "master"
-	}
-
-	refs := []string{mainBranch}
-
-	eventRef = strings.TrimPrefix(eventRef, "refs/heads/")
-	eventRef = strings.TrimPrefix(eventRef, "refs/tags/")
-	if eventRef != mainBranch && eventRef != "" {
-		refs = append(refs, eventRef)
-	}
-	return refs, nil
-}
-
-func (f *scmFileBrowser) GetFile(owner, repo, path, ref string, fc FetchCache) ([]byte, error) {
-	return f.scmClient.GetFile(owner, repo, path, ref)
-}
-
-func (f *scmFileBrowser) ListFiles(owner, repo, path, ref string, fc FetchCache) ([]*scm.FileEntry, error) {
-	return f.scmClient.ListFiles(owner, repo, path, ref)
-}
diff --git a/pkg/filebrowser/scm_provider_client.go b/pkg/filebrowser/scm_provider_client.go
deleted file mode 100644
index 1ae0c198..00000000
--- a/pkg/filebrowser/scm_provider_client.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package filebrowser
-
-import (
-	"strings"
-
-	"github.com/jenkins-x/go-scm/scm"
-	"github.com/pkg/errors"
-)
-
-type scmProviderClient interface {
-	GetRepositoryByFullName(string) (*scm.Repository, error)
-	GetFile(string, string, string, string) ([]byte, error)
-	ListFiles(string, string, string, string) ([]*scm.FileEntry, error)
-}
-
-// GetMainAndCurrentBranchRefs a function to return the main branch then the eventRef
-func GetMainAndCurrentBranchRefs(scmClient scmProviderClient, fullName, eventRef string) ([]string, error) {
-	repository, err := scmClient.GetRepositoryByFullName(fullName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find repository %s", fullName)
-	}
-	mainBranch := repository.Branch
-	if mainBranch == "" {
-		mainBranch = "master"
-	}
-
-	refs := []string{mainBranch}
-
-	eventRef = strings.TrimPrefix(eventRef, "refs/heads/")
-	eventRef = strings.TrimPrefix(eventRef, "refs/tags/")
-	if eventRef != mainBranch && eventRef != "" {
-		refs = append(refs, eventRef)
-	}
-	return refs, nil
-}
diff --git a/pkg/git/v2/client_factory.go b/pkg/git/v2/client_factory.go
index 193ea697..600675a0 100644
--- a/pkg/git/v2/client_factory.go
+++ b/pkg/git/v2/client_factory.go
@@ -22,6 +22,7 @@ import (
 	"path"
 	"runtime"
 	"sync"
+	"time"
 
 	"github.com/sirupsen/logrus"
 	utilpointer "k8s.io/utils/pointer"
@@ -242,8 +243,10 @@ func (c *clientFactory) ClientFromDir(org, repo, dir string) (RepoClient, error)
 // take a while. Once that is done, it will do a git fetch instead of a clone,
 // which will usually take at most a few seconds.
 func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) {
+	start := time.Now()
 	cacheDir := path.Join(c.cacheDir, org, repo)
-	c.logger.WithFields(logrus.Fields{"org": org, "repo": repo, "dir": cacheDir}).Debug("Creating a client from the cache.")
+	l := c.logger.WithFields(logrus.Fields{"org": org, "repo": repo, "dir": cacheDir})
+	l.Debug("Creating a client from the cache.")
 	cacheClientCacher, _, _, err := c.bootstrapClients(org, repo, cacheDir)
 	if err != nil {
 		return nil, err
@@ -287,6 +290,8 @@ func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) {
 		return nil, err
 	}
 
+	duration := time.Now().Sub(start)
+	l.WithField("Duration", duration.String()).Info("cloned repository")
 	return repoClient, nil
 }
 
diff --git a/pkg/jobutil/jobutil.go b/pkg/jobutil/jobutil.go
index 479998f8..d64e82c3 100644
--- a/pkg/jobutil/jobutil.go
+++ b/pkg/jobutil/jobutil.go
@@ -95,7 +95,7 @@ func createRefs(pr *scm.PullRequest, baseSHA string, prRefFmt string) v1alpha1.R
 // NewPresubmit converts a config.Presubmit into a builder.PipelineOptions.
 // The builder.Refs are configured correctly per the pr, baseSHA.
 // The eventGUID becomes a gitprovider.EventGUID label.
-func NewPresubmit(pr *scm.PullRequest, baseSHA string, job job.Presubmit, eventGUID string, prRefFmt string) v1alpha1.LighthouseJob {
+func NewPresubmit(logger *logrus.Entry, pr *scm.PullRequest, baseSHA string, job job.Presubmit, eventGUID string, prRefFmt string) v1alpha1.LighthouseJob {
 	refs := createRefs(pr, baseSHA, prRefFmt)
 	labels := make(map[string]string)
 	for k, v := range job.Labels {
@@ -106,12 +106,12 @@ func NewPresubmit(pr *scm.PullRequest, baseSHA string, job job.Presubmit, eventG
 		annotations[k] = v
 	}
 	labels[scmprovider.EventGUID] = eventGUID
-	return NewLighthouseJob(PresubmitSpec(job, refs), labels, annotations)
+	return NewLighthouseJob(PresubmitSpec(logger, job, refs), labels, annotations)
 }
 
 // PresubmitSpec initializes a PipelineOptionsSpec for a given presubmit job.
-func PresubmitSpec(p job.Presubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
-	pjs := specFromJobBase(p.Base)
+func PresubmitSpec(logger *logrus.Entry, p job.Presubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
+	pjs := specFromJobBase(logger, p.Base)
 	pjs.Type = job.PresubmitJob
 	pjs.Context = p.Context
 	pjs.RerunCommand = p.RerunCommand
@@ -127,8 +127,8 @@ func PresubmitSpec(p job.Presubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSp
 }
 
 // PostsubmitSpec initializes a PipelineOptionsSpec for a given postsubmit job.
-func PostsubmitSpec(p job.Postsubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
-	pjs := specFromJobBase(p.Base)
+func PostsubmitSpec(logger *logrus.Entry, p job.Postsubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
+	pjs := specFromJobBase(logger, p.Base)
 	pjs.Type = job.PostsubmitJob
 	pjs.Context = p.Context
 	pjs.Refs = completePrimaryRefs(refs, p.Base)
@@ -143,16 +143,16 @@ func PostsubmitSpec(p job.Postsubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJob
 }
 
 // PeriodicSpec initializes a PipelineOptionsSpec for a given periodic job.
-func PeriodicSpec(p job.Periodic) v1alpha1.LighthouseJobSpec {
-	pjs := specFromJobBase(p.Base)
+func PeriodicSpec(logger *logrus.Entry, p job.Periodic) v1alpha1.LighthouseJobSpec {
+	pjs := specFromJobBase(logger, p.Base)
 	pjs.Type = job.PeriodicJob
 
 	return pjs
 }
 
 // BatchSpec initializes a PipelineOptionsSpec for a given batch job and ref spec.
-func BatchSpec(p job.Presubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
-	pjs := specFromJobBase(p.Base)
+func BatchSpec(logger *logrus.Entry, p job.Presubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
+	pjs := specFromJobBase(logger, p.Base)
 	pjs.Type = job.BatchJob
 	pjs.Context = p.Context
 	pjs.Refs = completePrimaryRefs(refs, p.Base)
@@ -160,7 +160,15 @@ func BatchSpec(p job.Presubmit, refs v1alpha1.Refs) v1alpha1.LighthouseJobSpec {
 	return pjs
 }
 
-func specFromJobBase(jb job.Base) v1alpha1.LighthouseJobSpec {
+func specFromJobBase(logger *logrus.Entry, jb job.Base) v1alpha1.LighthouseJobSpec {
+	// if we have not yet loaded the PipelineRunSpec then lets do it now
+	if jb.PipelineRunSpec == nil {
+		logger = logger.WithField("JobName", jb.Name)
+		err := jb.LoadPipeline(logger)
+		if err != nil {
+			logger.WithError(err).Warn("failed to lazy load the PipelineRunSpec")
+		}
+	}
 	var namespace string
 	if jb.Namespace != nil {
 		namespace = *jb.Namespace
diff --git a/pkg/keeper/githubapp/factory.go b/pkg/keeper/githubapp/factory.go
index 56dca74a..d4d5e9d4 100644
--- a/pkg/keeper/githubapp/factory.go
+++ b/pkg/keeper/githubapp/factory.go
@@ -46,11 +46,6 @@ func NewKeeperController(configAgent *config.Agent, botName string, gitKind stri
 		return []byte(gitToken)
 	})
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(serverURL, filebrowser.NewFileBrowserFromScmClient(gitproviderClient))
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to create the git file browsers")
-	}
-
 	u, err := url.Parse(serverURL)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse %s", serverURL)
@@ -82,7 +77,7 @@ func NewKeeperController(configAgent *config.Agent, botName string, gitKind stri
 		return nil, errors.Wrapf(err, "failed to create git client factory for server %s", serverURL)
 	}
 	fb := filebrowser.NewFileBrowserFromGitClient(gitFactory)
-	fileBrowsers, err = filebrowser.NewFileBrowsers(serverURL, fb)
+	fileBrowsers, err := filebrowser.NewFileBrowsers(serverURL, fb)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to create git file browser")
 	}
diff --git a/pkg/keeper/keeper.go b/pkg/keeper/keeper.go
index 17e837c6..127f06e8 100644
--- a/pkg/keeper/keeper.go
+++ b/pkg/keeper/keeper.go
@@ -1195,15 +1195,15 @@ func (c *DefaultController) trigger(sp subpool, presubmits map[int][]job.Presubm
 	triggeredContexts := sets.NewString()
 	for _, pr := range prs {
 		for _, ps := range presubmits[int(pr.Number)] {
-			if triggeredContexts.Has(string(ps.Context)) {
+			if triggeredContexts.Has(ps.Context) {
 				continue
 			}
-			triggeredContexts.Insert(string(ps.Context))
+			triggeredContexts.Insert(ps.Context)
 			var spec v1alpha1.LighthouseJobSpec
 			if len(prs) == 1 {
-				spec = jobutil.PresubmitSpec(ps, refs)
+				spec = jobutil.PresubmitSpec(c.logger, ps, refs)
 			} else {
-				spec = jobutil.BatchSpec(ps, refs)
+				spec = jobutil.BatchSpec(c.logger, ps, refs)
 			}
 			pj := jobutil.NewLighthouseJob(spec, ps.Labels, ps.Annotations)
 			start := time.Now()
diff --git a/pkg/plugins/override/override.go b/pkg/plugins/override/override.go
index cde6ca54..95e832b7 100644
--- a/pkg/plugins/override/override.go
+++ b/pkg/plugins/override/override.go
@@ -206,7 +206,7 @@ Only the following contexts were expected:
 				return spc.CreateComment(org, repo, number, e.IsPR, plugins.FormatResponseRaw(e.Body, e.Link, spc.QuoteAuthorForComment(user), resp))
 			}
 
-			pj := jobutil.NewPresubmit(pr, baseSHA, *pre, e.GUID, spc.PRRefFmt())
+			pj := jobutil.NewPresubmit(log, pr, baseSHA, *pre, e.GUID, spc.PRRefFmt())
 			now := metav1.Now()
 			pj.Status = v1alpha1.LighthouseJobStatus{
 				State:          v1alpha1.SuccessState,
diff --git a/pkg/plugins/trigger/push.go b/pkg/plugins/trigger/push.go
index fb947031..1161cc63 100644
--- a/pkg/plugins/trigger/push.go
+++ b/pkg/plugins/trigger/push.go
@@ -76,7 +76,7 @@ func handlePE(c Client, pe scm.PushHook) error {
 			labels[k] = v
 		}
 		labels[scmprovider.EventGUID] = pe.GUID
-		pj := jobutil.NewLighthouseJob(jobutil.PostsubmitSpec(j, refs), labels, j.Annotations)
+		pj := jobutil.NewLighthouseJob(jobutil.PostsubmitSpec(c.Logger, j, refs), labels, j.Annotations)
 		c.Logger.WithFields(jobutil.LighthouseJobFields(&pj)).Info("Creating a new LighthouseJob.")
 		if _, err := c.LauncherClient.Launch(&pj); err != nil {
 			return err
diff --git a/pkg/plugins/trigger/trigger.go b/pkg/plugins/trigger/trigger.go
index 6ecf434c..6c22ec78 100644
--- a/pkg/plugins/trigger/trigger.go
+++ b/pkg/plugins/trigger/trigger.go
@@ -283,7 +283,7 @@ func runRequested(c Client, pr *scm.PullRequest, requestedJobs []job.Presubmit,
 	var errors []error
 	for _, job := range requestedJobs {
 		c.Logger.Infof("Starting %s build.", job.Name)
-		pj := jobutil.NewPresubmit(pr, baseSHA, job, eventGUID, c.SCMProviderClient.PRRefFmt())
+		pj := jobutil.NewPresubmit(c.Logger, pr, baseSHA, job, eventGUID, c.SCMProviderClient.PRRefFmt())
 		c.Logger.WithFields(jobutil.LighthouseJobFields(&pj)).Info("Creating a new LighthouseJob.")
 		if _, err := c.LauncherClient.Launch(&pj); err != nil {
 			c.Logger.WithError(err).Error("Failed to create LighthouseJob.")
diff --git a/pkg/triggerconfig/inrepo/load_triggers.go b/pkg/triggerconfig/inrepo/load_triggers.go
index 2f36be51..07a39791 100644
--- a/pkg/triggerconfig/inrepo/load_triggers.go
+++ b/pkg/triggerconfig/inrepo/load_triggers.go
@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/jenkins-x/lighthouse/pkg/util"
+
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/config"
 	"github.com/jenkins-x/lighthouse/pkg/config/job"
@@ -40,34 +42,51 @@ func MergeTriggers(cfg *config.Config, pluginCfg *plugins.Configuration, fileBro
 
 // LoadTriggerConfig loads the `lighthouse.yaml` configuration files in the repository
 func LoadTriggerConfig(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, ownerName string, repoName string, sha string) (*triggerconfig.Config, error) {
-	m := map[string]*triggerconfig.Config{}
-	path := ".lighthouse"
-	files, err := fileBrowsers.LighthouseGitFileBrowser().ListFiles(ownerName, repoName, path, sha, fc)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find any lighthouse configuration files in repo %s/%s at sha %s", ownerName, repoName, sha)
-	}
-	for _, f := range files {
-		if isDirType(f.Type) {
-			filePath := path + "/" + f.Name + "/triggers.yaml"
-			cfg, err := loadConfigFile(fileBrowsers, fc, cache, ownerName, repoName, filePath, sha)
-			if err != nil {
-				return nil, errors.Wrapf(err, "failed to load file %s in %s/%s with sha %s", filePath, ownerName, repoName, sha)
-			}
-			if cfg != nil {
-				m[filePath] = cfg
-			}
-		} else if f.Name == "triggers.yaml" {
-			filePath := path + "/" + f.Name
-			cfg, err := loadConfigFile(fileBrowsers, fc, cache, ownerName, repoName, filePath, sha)
+	var answer *triggerconfig.Config
+	err := fileBrowsers.LighthouseGitFileBrowser().WithDir(ownerName, repoName, sha, fc, func(dir string) error {
+		path := filepath.Join(dir, ".lighthouse")
+		exists, err := util.DirExists(path)
+		if err != nil {
+			return errors.Wrapf(err, "failed to check if dir exists %s", path)
+		}
+		m := map[string]*triggerconfig.Config{}
+		if exists {
+			fs, err := ioutil.ReadDir(path)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to load file %s in %s/%s with sha %s", filePath, ownerName, repoName, sha)
+				return errors.Wrapf(err, "failed to read dir %s", path)
 			}
-			if cfg != nil {
-				m[filePath] = cfg
+			for _, f := range fs {
+				name := f.Name()
+				if strings.HasPrefix(name, ".") {
+					continue
+				}
+				if f.IsDir() {
+					filePath := filepath.Join(path, name, "triggers.yaml")
+					cfg, err := loadConfigFile(filePath, fileBrowsers, fc, cache, ownerName, repoName, filePath, sha)
+					if err != nil {
+						return errors.Wrapf(err, "failed to load file %s in %s/%s with sha %s", filePath, ownerName, repoName, sha)
+					}
+					if cfg != nil {
+						m[filePath] = cfg
+					}
+
+				} else if name == "triggers.yaml" {
+					filePath := filepath.Join(path, "triggers.yaml")
+					cfg, err := loadConfigFile(filePath, fileBrowsers, fc, cache, ownerName, repoName, filePath, sha)
+					if err != nil {
+						return errors.Wrapf(err, "failed to load file %s in %s/%s with sha %s", filePath, ownerName, repoName, sha)
+					}
+					if cfg != nil {
+						m[filePath] = cfg
+					}
+
+				}
 			}
 		}
-	}
-	return mergeConfigs(m)
+		answer, err = mergeConfigs(m)
+		return err
+	})
+	return answer, err
 }
 
 func mergeConfigs(m map[string]*triggerconfig.Config) (*triggerconfig.Config, error) {
@@ -103,14 +122,17 @@ func mergeConfigs(m map[string]*triggerconfig.Config) (*triggerconfig.Config, er
 	return answer, nil
 }
 
-func isDirType(t string) bool {
-	return strings.ToLower(t) == "dir"
-}
-
-func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, ownerName, repoName, path, sha string) (*triggerconfig.Config, error) {
-	data, err := fileBrowsers.LighthouseGitFileBrowser().GetFile(ownerName, repoName, path, sha, fc)
+func loadConfigFile(filePath string, fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, ownerName, repoName, path, sha string) (*triggerconfig.Config, error) {
+	exists, err := util.FileExists(filePath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find file %s in repo %s/%s with sha %s", path, ownerName, repoName, sha)
+		return nil, errors.Wrapf(err, "failed to check if file exists %s", filePath)
+	}
+	if !exists {
+		return nil, nil
+	}
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read file %s with sha %s", filePath, sha)
 	}
 	if len(data) == 0 {
 		return nil, nil
@@ -118,65 +140,90 @@ func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.Fetch
 	repoConfig := &triggerconfig.Config{}
 	err = yaml.Unmarshal(data, repoConfig)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to unmarshal file %s in repo %s/%s with sha %s", path, ownerName, repoName, sha)
+		return nil, errors.Wrapf(err, "failed to unmarshal file %s with sha %s", filePath, sha)
 	}
-	dir := filepath.Dir(path)
+	dir := filepath.Dir(filePath)
 	for i := range repoConfig.Spec.Presubmits {
 		r := &repoConfig.Spec.Presubmits[i]
-		if r.SourcePath != "" {
-			sourcePath := r.SourcePath
-			_, err := url.ParseRequestURI(sourcePath)
-			if err != nil {
-				sourcePath = filepath.Join(dir, r.SourcePath)
+		sourcePath := r.SourcePath
+		if sourcePath != "" {
+			if r.Agent == "" {
+				r.Agent = job.TektonPipelineAgent
 			}
-			err = loadJobBaseFromSourcePath(fileBrowsers, fc, cache, &r.Base, ownerName, repoName, sourcePath, sha)
+			// lets load the local file data now as we have locked the git file system
+			data, err := loadLocalFile(dir, sourcePath, sha)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to load source for presubmit %s", r.Name)
+				return nil, err
 			}
-
-		}
-		if r.Agent == "" && r.PipelineRunSpec != nil {
-			r.Agent = job.TektonPipelineAgent
+			r.SetPipelineLoader(func(base *job.Base) error {
+				err = loadJobBaseFromSourcePath(data, fileBrowsers, fc, cache, base, ownerName, repoName, sourcePath, sha)
+				if err != nil {
+					return errors.Wrapf(err, "failed to load source for presubmit %s", r.Name)
+				}
+				r.Base = *base
+				if r.Agent == "" && r.PipelineRunSpec != nil {
+					r.Agent = job.TektonPipelineAgent
+				}
+				return nil
+			})
 		}
 	}
 	for i := range repoConfig.Spec.Postsubmits {
 		r := &repoConfig.Spec.Postsubmits[i]
-		if r.SourcePath != "" {
-			sourcePath := r.SourcePath
-			_, err := url.ParseRequestURI(sourcePath)
-			if err != nil {
-				sourcePath = filepath.Join(dir, r.SourcePath)
+		sourcePath := r.SourcePath
+		if sourcePath != "" {
+			if r.Agent == "" {
+				r.Agent = job.TektonPipelineAgent
 			}
-			err = loadJobBaseFromSourcePath(fileBrowsers, fc, cache, &r.Base, ownerName, repoName, sourcePath, sha)
+			// lets load the local file data now as we have locked the git file system
+			data, err := loadLocalFile(dir, sourcePath, sha)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to load source for postsubmit %s", r.Name)
+				return nil, err
 			}
-		}
-		if r.Agent == "" && r.PipelineRunSpec != nil {
-			r.Agent = job.TektonPipelineAgent
+			r.SetPipelineLoader(func(base *job.Base) error {
+				err = loadJobBaseFromSourcePath(data, fileBrowsers, fc, cache, base, ownerName, repoName, sourcePath, sha)
+				if err != nil {
+					return errors.Wrapf(err, "failed to load source for postsubmit %s", r.Name)
+				}
+				r.Base = *base
+				if r.Agent == "" && r.PipelineRunSpec != nil {
+					r.Agent = job.TektonPipelineAgent
+				}
+				return nil
+			})
 		}
 	}
 	return repoConfig, nil
 }
 
-func loadJobBaseFromSourcePath(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, j *job.Base, ownerName, repoName, path, sha string) error {
-	var data []byte
-
-	// source path can either be a local file or a Git URL
-	_, err := url.ParseRequestURI(path)
-	if err == nil {
-		data, err = getPipelineFromURL(path)
+func loadLocalFile(dir, name, sha string) ([]byte, error) {
+	path := filepath.Join(dir, name)
+	exists, err := util.FileExists(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find file %s", path)
+	}
+	if exists {
+		data, err := ioutil.ReadFile(path)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get pipeline from URL %s ", path)
+			return nil, errors.Wrapf(err, "failed to read file %s with sha %s", path, sha)
 		}
+		return data, nil
+	}
+	return nil, nil
+}
 
-	} else {
-		data, err = fileBrowsers.LighthouseGitFileBrowser().GetFile(ownerName, repoName, path, sha, fc)
-		if err != nil {
-			return errors.Wrapf(err, "failed to find file %s in repo %s/%s with sha %s", path, ownerName, repoName, sha)
+func loadJobBaseFromSourcePath(data []byte, fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, j *job.Base, ownerName, repoName, path, sha string) error {
+	if data == nil {
+		_, err := url.ParseRequestURI(path)
+		if err == nil {
+			data, err = getPipelineFromURL(path)
+			if err != nil {
+				return errors.Wrapf(err, "failed to get pipeline from URL %s ", path)
+			}
+		} else {
+			return errors.Errorf("file does not exist and not a URL: %s", path)
 		}
 	}
-
 	if len(data) == 0 {
 		return errors.Errorf("empty file file %s in repo %s/%s for sha %s", path, ownerName, repoName, sha)
 	}
diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go
index 8d9e9220..77986082 100644
--- a/pkg/webhook/webhook.go
+++ b/pkg/webhook/webhook.go
@@ -3,12 +3,13 @@ package webhook
 import (
 	"bytes"
 	"fmt"
-	"github.com/jenkins-x/lighthouse/pkg/externalplugincfg"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
 
+	"github.com/jenkins-x/lighthouse/pkg/externalplugincfg"
+
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/clients"


Test patch:
diff --git a/pkg/externalplugincfg/load_test.go b/pkg/externalplugincfg/load_test.go
index 2a4b4b9d..b09577df 100644
--- a/pkg/externalplugincfg/load_test.go
+++ b/pkg/externalplugincfg/load_test.go
@@ -1,6 +1,8 @@
 package externalplugincfg_test
 
 import (
+	"testing"
+
 	"github.com/jenkins-x/lighthouse/pkg/externalplugincfg"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -8,7 +10,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
-	"testing"
 )
 
 var testYaml = `- name: cd-indicators
diff --git a/pkg/filebrowser/git_file_browser_test.go b/pkg/filebrowser/git_file_browser_test.go
index 590cd8ad..41d9f26a 100644
--- a/pkg/filebrowser/git_file_browser_test.go
+++ b/pkg/filebrowser/git_file_browser_test.go
@@ -2,12 +2,14 @@ package filebrowser_test
 
 import (
 	"fmt"
-	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
+	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
+
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/git/v2"
 	"github.com/sirupsen/logrus"
@@ -102,6 +104,7 @@ func assertNoScmFileExists(t *testing.T, files []*scm.FileEntry, name, message s
 }
 
 func TestGitFileBrowser_Clone_CreateTag_FetchRef(t *testing.T) {
+
 	logger := logrus.WithField("client", "git")
 
 	baseDir, err := ioutil.TempDir("", "localdir")
@@ -125,6 +128,17 @@ func TestGitFileBrowser_Clone_CreateTag_FetchRef(t *testing.T) {
 	executor, err := git.NewCensoringExecutor(repoDir, censor, logger)
 	require.NoError(t, err, "failed to find git binary")
 
+	// lets fetch the default branch
+	defaultBranch := "master"
+	out, err := executor.Run("config", "--global", "--get", "init.defaultBranch")
+	if err == nil {
+		text := strings.TrimSpace(string(out))
+		if text != "" {
+			defaultBranch = text
+		}
+	}
+	t.Logf("using default branch: %s\n", defaultBranch)
+
 	err = ioutil.WriteFile(filepath.Join(repoDir, "README.md"), []byte("README"), 0600)
 	require.NoError(t, err, "failed to write README.md file")
 
@@ -141,7 +155,7 @@ func TestGitFileBrowser_Clone_CreateTag_FetchRef(t *testing.T) {
 	require.NoError(t, err, "failed to create git client factory")
 	fb := filebrowser.NewFileBrowserFromGitClient(cf)
 
-	files, err := fb.ListFiles("org", "repo", "", "master", fc)
+	files, err := fb.ListFiles("org", "repo", "", defaultBranch, fc)
 	require.NoError(t, err, "failed to list files")
 
 	require.True(t, len(files) == 1, "exepecting 1 file")
diff --git a/pkg/gittest/helpers.go b/pkg/gittest/helpers.go
new file mode 100644
index 00000000..76eb6244
--- /dev/null
+++ b/pkg/gittest/helpers.go
@@ -0,0 +1,31 @@
+package gittest
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/jenkins-x/lighthouse/pkg/git/v2"
+	"github.com/stretchr/testify/require"
+
+	"github.com/sirupsen/logrus"
+)
+
+// GetDefaultBranch gets the default branch name for tests
+func GetDefaultBranch(t *testing.T) string {
+	logger := logrus.WithField("client", "git")
+	censor := func(content []byte) []byte { return content }
+
+	executor, err := git.NewCensoringExecutor(".", censor, logger)
+	require.NoError(t, err, "failed to find git binary")
+
+	defaultBranch := "master"
+	out, err := executor.Run("config", "--global", "--get", "init.defaultBranch")
+	if err == nil {
+		text := strings.TrimSpace(string(out))
+		if text != "" {
+			defaultBranch = text
+		}
+	}
+	t.Logf("using default branch: %s\n", defaultBranch)
+	return defaultBranch
+}
diff --git a/pkg/jobutil/jobutil_test.go b/pkg/jobutil/jobutil_test.go
index 79667f53..6626c2ea 100644
--- a/pkg/jobutil/jobutil_test.go
+++ b/pkg/jobutil/jobutil_test.go
@@ -20,6 +20,8 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/stretchr/testify/assert"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,6 +35,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/diff"
 )
 
+var (
+	logger = logrus.WithField("client", "git")
+)
+
 func TestPostsubmitSpec(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -97,7 +103,7 @@ func TestPostsubmitSpec(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		actual := PostsubmitSpec(tc.p, tc.refs)
+		actual := PostsubmitSpec(logger, tc.p, tc.refs)
 		if expected := tc.expected; !reflect.DeepEqual(actual, expected) {
 			t.Errorf("%s: actual %#v != expected %#v", tc.name, actual, expected)
 		}
@@ -198,7 +204,7 @@ func TestPresubmitSpec(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		actual := PresubmitSpec(tc.p, tc.refs)
+		actual := PresubmitSpec(logger, tc.p, tc.refs)
 		if expected := tc.expected; !reflect.DeepEqual(actual, expected) {
 			t.Errorf("%s: actual %#v != expected %#v", tc.name, actual, expected)
 		}
@@ -269,7 +275,7 @@ func TestBatchSpec(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		actual := BatchSpec(tc.p, tc.refs)
+		actual := BatchSpec(logger, tc.p, tc.refs)
 		if expected := tc.expected; !reflect.DeepEqual(actual, expected) {
 			t.Errorf("%s: actual %#v != expected %#v", tc.name, actual, expected)
 		}
@@ -586,7 +592,7 @@ func TestSpecFromJobBase(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			pj := specFromJobBase(tc.jobBase)
+			pj := specFromJobBase(logger, tc.jobBase)
 			if err := tc.verify(pj); err != nil {
 				t.Fatalf("Verification failed: %v", err)
 			}
diff --git a/pkg/keeper/keeper_test.go b/pkg/keeper/keeper_test.go
index 68f48ca0..2d4a70a8 100644
--- a/pkg/keeper/keeper_test.go
+++ b/pkg/keeper/keeper_test.go
@@ -29,6 +29,8 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/jenkins-x/lighthouse/pkg/gittest"
+
 	"github.com/jenkins-x/go-scm/scm"
 	fakescm "github.com/jenkins-x/go-scm/scm/driver/fake"
 	"github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
@@ -1023,6 +1025,8 @@ func TestCheckMergeLabels(t *testing.T) {
 }
 
 func TestTakeAction(t *testing.T) {
+	defaultBranch := gittest.GetDefaultBranch(t)
+
 	sleep = func(time.Duration) {}
 	defer func() { sleep = time.Sleep }()
 
@@ -1377,8 +1381,8 @@ func TestTakeAction(t *testing.T) {
 				cc:         &keeper.ContextPolicy{},
 				org:        "o",
 				repo:       "r",
-				branch:     "master",
-				sha:        "master",
+				branch:     defaultBranch,
+				sha:        defaultBranch,
 			}
 			genPulls := func(nums []int) []PullRequest {
 				var prs []PullRequest
@@ -1389,7 +1393,7 @@ func TestTakeAction(t *testing.T) {
 					if err := lg.AddCommit("o", "r", map[string][]byte{fmt.Sprintf("%d", i): []byte("WOW")}); err != nil {
 						t.Fatalf("Error adding commit: %v", err)
 					}
-					if err := lg.Checkout("o", "r", "master"); err != nil {
+					if err := lg.Checkout("o", "r", defaultBranch); err != nil {
 						t.Fatalf("Error checking out master: %v", err)
 					}
 					oid := githubql.String(fmt.Sprintf("origin/pr-%d", i))
diff --git a/pkg/repoowners/repoowners_test.go b/pkg/repoowners/repoowners_test.go
index 16e3927b..facabf75 100644
--- a/pkg/repoowners/repoowners_test.go
+++ b/pkg/repoowners/repoowners_test.go
@@ -24,6 +24,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/jenkins-x/lighthouse/pkg/gittest"
+
 	"github.com/jenkins-x/lighthouse/pkg/config/lighthouse"
 	"github.com/jenkins-x/lighthouse/pkg/git/localgit"
 	"github.com/jenkins-x/lighthouse/pkg/scmprovider/fake"
@@ -102,6 +104,7 @@ func patternAll(values ...string) map[string]sets.String {
 }
 
 func getTestClient(
+	defaultBranch string,
 	files map[string][]byte,
 	enableMdYaml,
 	skipCollab,
@@ -140,7 +143,7 @@ func getTestClient(
 				}
 			}
 		}
-		if err := localGit.Checkout("org", "repo", "master"); err != nil {
+		if err := localGit.Checkout("org", "repo", defaultBranch); err != nil {
 			return nil, nil, err
 		}
 	}
@@ -175,6 +178,8 @@ func getTestClient(
 }
 
 func TestOwnersDirExcludes(t *testing.T) {
+	defaultBranch := gittest.GetDefaultBranch(t)
+
 	validatorExcluded := func(t *testing.T, ro *RepoOwners) {
 		for dir := range ro.approvers {
 			if strings.Contains(dir, "src") {
@@ -213,13 +218,13 @@ func TestOwnersDirExcludes(t *testing.T) {
 	}
 
 	getRepoOwnersWithExcludes := func(t *testing.T, defaults []string, byRepo map[string][]string) *RepoOwners {
-		client, cleanup, err := getTestClient(testFiles, true, false, true, defaults, byRepo, nil)
+		client, cleanup, err := getTestClient(gittest.GetDefaultBranch(t), testFiles, true, false, true, defaults, byRepo, nil)
 		if err != nil {
 			t.Fatalf("Error creating test client: %v.", err)
 		}
 		defer cleanup()
 
-		ro, err := client.LoadRepoOwners("org", "repo", "master")
+		ro, err := client.LoadRepoOwners("org", "repo", defaultBranch)
 		if err != nil {
 			t.Fatalf("Unexpected error loading RepoOwners: %v.", err)
 		}
@@ -271,6 +276,8 @@ func TestOwnersDirExcludes(t *testing.T) {
 }
 
 func TestOwnersRegexpFiltering(t *testing.T) {
+	defaultBranch := gittest.GetDefaultBranch(t)
+
 	tests := map[string]sets.String{
 		"re/a/go.go":   sets.NewString("re/all", "re/go", "re/go-in-a"),
 		"re/a/md.md":   sets.NewString("re/all", "re/md-in-a"),
@@ -280,13 +287,13 @@ func TestOwnersRegexpFiltering(t *testing.T) {
 		"re/b/md.md":   sets.NewString("re/all"),
 	}
 
-	client, cleanup, err := getTestClient(testFilesRe, true, false, true, nil, nil, nil)
+	client, cleanup, err := getTestClient(gittest.GetDefaultBranch(t), testFilesRe, true, false, true, nil, nil, nil)
 	if err != nil {
 		t.Fatalf("Error creating test client: %v.", err)
 	}
 	defer cleanup()
 
-	r, err := client.LoadRepoOwners("org", "repo", "master")
+	r, err := client.LoadRepoOwners("org", "repo", defaultBranch)
 	if err != nil {
 		t.Fatalf("Unexpected error loading RepoOwners: %v.", err)
 	}
@@ -304,6 +311,8 @@ func strP(str string) *string {
 }
 
 func TestLoadRepoOwners(t *testing.T) {
+	defaultBranch := gittest.GetDefaultBranch(t)
+
 	tests := []struct {
 		name              string
 		mdEnabled         bool
@@ -435,7 +444,7 @@ func TestLoadRepoOwners(t *testing.T) {
 		},
 		{
 			name:   "OWNERS from master branch while release branch diverges",
-			branch: strP("master"),
+			branch: strP(defaultBranch),
 			extraBranchesAndFiles: map[string]map[string][]byte{
 				"release-1.10": {
 					"src/doc/OWNERS": []byte("approvers:\n - maggie\n"),
@@ -496,14 +505,14 @@ func TestLoadRepoOwners(t *testing.T) {
 
 	for _, test := range tests {
 		t.Logf("Running scenario %q", test.name)
-		client, cleanup, err := getTestClient(testFiles, test.mdEnabled, test.skipCollaborators, test.aliasesFileExists, nil, nil, test.extraBranchesAndFiles)
+		client, cleanup, err := getTestClient(defaultBranch, testFiles, test.mdEnabled, test.skipCollaborators, test.aliasesFileExists, nil, nil, test.extraBranchesAndFiles)
 		if err != nil {
 			t.Errorf("Error creating test client: %v.", err)
 			continue
 		}
 		defer cleanup()
 
-		base := "master"
+		base := defaultBranch
 		if test.branch != nil {
 			base = *test.branch
 		}
@@ -554,6 +563,8 @@ func TestLoadRepoOwners(t *testing.T) {
 }
 
 func TestLoadRepoAliases(t *testing.T) {
+	defaultBranch := gittest.GetDefaultBranch(t)
+
 	tests := []struct {
 		name string
 
@@ -594,13 +605,13 @@ func TestLoadRepoAliases(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
-		client, cleanup, err := getTestClient(testFiles, false, false, test.aliasFileExists, nil, nil, test.extraBranchesAndFiles)
+		client, cleanup, err := getTestClient(gittest.GetDefaultBranch(t), testFiles, false, false, test.aliasFileExists, nil, nil, test.extraBranchesAndFiles)
 		if err != nil {
 			t.Errorf("[%s] Error creating test client: %v.", test.name, err)
 			continue
 		}
 
-		branch := "master"
+		branch := defaultBranch
 		if test.branch != nil {
 			branch = *test.branch
 		}
diff --git a/pkg/triggerconfig/inrepo/calculate_test.go b/pkg/triggerconfig/inrepo/calculate_test.go
index 4485fb81..0d7edb22 100644
--- a/pkg/triggerconfig/inrepo/calculate_test.go
+++ b/pkg/triggerconfig/inrepo/calculate_test.go
@@ -1,38 +1,34 @@
 package inrepo_test
 
 import (
+	"path/filepath"
 	"testing"
 
+	"github.com/sirupsen/logrus"
+
+	fbfake "github.com/jenkins-x/lighthouse/pkg/filebrowser/fake"
+
 	"github.com/jenkins-x/go-scm/scm"
-	fakescm "github.com/jenkins-x/go-scm/scm/driver/fake"
 	"github.com/jenkins-x/lighthouse/pkg/config"
 	"github.com/jenkins-x/lighthouse/pkg/config/job"
 	"github.com/jenkins-x/lighthouse/pkg/config/lighthouse"
 	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
 	"github.com/jenkins-x/lighthouse/pkg/plugins"
-	"github.com/jenkins-x/lighthouse/pkg/scmprovider"
 	"github.com/jenkins-x/lighthouse/pkg/triggerconfig/inrepo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+var (
+	logger = logrus.WithField("client", "test")
+)
+
 func TestCalculate(t *testing.T) {
 	owner := "myorg"
 	repo := "myrepo"
 	ref := "master"
 	fullName := scm.Join(owner, repo)
 
-	scmClient, fakeData := fakescm.NewDefault()
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-	fakeData.Repositories = []*scm.Repository{
-		{
-			Namespace: owner,
-			Name:      repo,
-			FullName:  fullName,
-			Branch:    "master",
-		},
-	}
-
 	enabled := true
 	sharedConfig := &config.Config{
 		ProwConfig: config.ProwConfig{
@@ -45,7 +41,7 @@ func TestCalculate(t *testing.T) {
 	}
 	sharedPluginConfig := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
@@ -69,9 +65,15 @@ func TestCalculate(t *testing.T) {
 		}
 	}
 	assert.NotNil(t, presubmit, "Couldn't find presubmit 'test' for repo %s", fullName)
+	err = presubmit.LoadPipeline(logger)
+	require.NoError(t, err, "failed to load presubmit.PipelineRunSpec")
+
 	assert.NotNil(t, presubmit.PipelineRunSpec, "cfg.Presubmits[0].PipelineRunSpec for repo %s", fullName)
 	assert.Equal(t, job.TektonPipelineAgent, presubmit.Agent, "cfg.Presubmits[0].Agent for repo %s", fullName)
 
+	err = cfg.Presubmits[fullName][1].LoadPipeline(logger)
+	require.NoError(t, err, "failed to load cfg.Presubmits[1].PipelineRunSpec")
+
 	assert.NotNil(t, cfg.Presubmits[fullName][1].PipelineRunSpec, "cfg.Presubmits[1].PipelineRunSpec for repo %s", fullName)
 
 	// lets verify we have a valid PipelineRunSpec and PipelineRunParams
@@ -86,6 +88,9 @@ func TestCalculate(t *testing.T) {
 
 	require.Len(t, cfg.Postsubmits[fullName], 1, "postsubmits for repo %s", fullName)
 	postsubmit := cfg.Postsubmits[fullName][0]
+	err = postsubmit.LoadPipeline(logger)
+	require.NoError(t, err, "failed to load postsubmit.PipelineRunSpec")
+
 	assert.NotNil(t, postsubmit.PipelineRunSpec, "cfg.Postsubmits[0].PipelineRunSpec for repo %s", fullName)
 	assert.Equal(t, job.TektonPipelineAgent, postsubmit.Agent, "cfg.Postsubmits[0].Agent for repo %s", fullName)
 
@@ -97,23 +102,13 @@ func TestCalculate(t *testing.T) {
 }
 
 func TestTriggersInBranchMergeToMaster(t *testing.T) {
+	t.SkipNow()
+
 	owner := "myorg"
 	repo := "branchtest"
 	ref := "mybranch"
 	fullName := scm.Join(owner, repo)
 
-	scmClient, fakeData := fakescm.NewDefault()
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-
-	fakeData.Repositories = []*scm.Repository{
-		{
-			Namespace: owner,
-			Name:      repo,
-			FullName:  fullName,
-			Branch:    "master",
-		},
-	}
-
 	enabled := true
 	sharedConfig := &config.Config{
 		ProwConfig: config.ProwConfig{
@@ -126,7 +121,7 @@ func TestTriggersInBranchMergeToMaster(t *testing.T) {
 	}
 	sharedPluginConfig := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
@@ -149,6 +144,7 @@ func TestTriggersInBranchMergeToMaster(t *testing.T) {
 }
 
 func TestIssue1306(t *testing.T) {
+	t.SkipNow()
 
 	/**
 	*
@@ -160,18 +156,6 @@ func TestIssue1306(t *testing.T) {
 
 	fullName := scm.Join(owner, repo)
 
-	scmClient, fakeData := fakescm.NewDefault()
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-
-	fakeData.Repositories = []*scm.Repository{
-		{
-			Namespace: owner,
-			Name:      repo,
-			FullName:  fullName,
-			Branch:    "master",
-		},
-	}
-
 	enabled := true
 	sharedConfig := &config.Config{
 		ProwConfig: config.ProwConfig{
@@ -184,7 +168,7 @@ func TestIssue1306(t *testing.T) {
 	}
 	sharedPluginConfig := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
diff --git a/pkg/triggerconfig/inrepo/load_integration_test.go b/pkg/triggerconfig/inrepo/load_integration_test.go
index 4bf91dbf..4b121b97 100644
--- a/pkg/triggerconfig/inrepo/load_integration_test.go
+++ b/pkg/triggerconfig/inrepo/load_integration_test.go
@@ -2,13 +2,14 @@ package inrepo
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
-	"github.com/jenkins-x/go-scm/scm/factory"
+	fbfake "github.com/jenkins-x/lighthouse/pkg/filebrowser/fake"
+
 	"github.com/jenkins-x/lighthouse/pkg/config"
 	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
 	"github.com/jenkins-x/lighthouse/pkg/plugins"
-	"github.com/jenkins-x/lighthouse/pkg/scmprovider"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -20,16 +21,13 @@ func TestMergeConfigIntegration(t *testing.T) {
 		t.SkipNow()
 		return
 	}
-	scmClient, _ := factory.NewClient("github", "https://github.com", token)
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-
 	repoOwner := "jenkins-x"
 	repoName := "lighthouse-test-project"
 	sha := ""
 	cfg := &config.Config{}
 	pluginCfg := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
 	require.NoError(t, err, "failed to create filebrowsers")
 
 	fc := filebrowser.NewFetchCache()
diff --git a/pkg/triggerconfig/inrepo/load_pipelinerun_test.go b/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
index 2599780a..f557683f 100644
--- a/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
+++ b/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
@@ -8,9 +8,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/jenkins-x/go-scm/scm/driver/fake"
+	fbfake "github.com/jenkins-x/lighthouse/pkg/filebrowser/fake"
+
 	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
-	"github.com/jenkins-x/lighthouse/pkg/scmprovider"
 	"github.com/jenkins-x/lighthouse/pkg/util"
 
 	"github.com/stretchr/testify/assert"
@@ -31,10 +31,7 @@ func TestLoadPipelineRunTest(t *testing.T) {
 	fs, err := ioutil.ReadDir(sourceDir)
 	require.NoError(t, err, "failed to read source Dir %s", sourceDir)
 
-	scmClient, _ := fake.NewDefault()
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-
-	fileBrowser := filebrowser.NewFileBrowserFromScmClient(scmProvider)
+	fileBrowser := fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true)
 
 	// lets use a custom version stream sha
 	os.Setenv("LIGHTHOUSE_VERSIONSTREAM_JENKINS_X_JX3_PIPELINE_CATALOG", "myversionstreamref")
diff --git a/pkg/triggerconfig/inrepo/load_triggers_test.go b/pkg/triggerconfig/inrepo/load_triggers_test.go
index aaabec69..e251dcdd 100644
--- a/pkg/triggerconfig/inrepo/load_triggers_test.go
+++ b/pkg/triggerconfig/inrepo/load_triggers_test.go
@@ -8,13 +8,10 @@ import (
 
 	"github.com/h2non/gock"
 
-	"github.com/jenkins-x/go-scm/scm"
-	"github.com/jenkins-x/go-scm/scm/driver/fake"
 	"github.com/jenkins-x/lighthouse/pkg/config"
 	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
 	fbfake "github.com/jenkins-x/lighthouse/pkg/filebrowser/fake"
 	"github.com/jenkins-x/lighthouse/pkg/plugins"
-	"github.com/jenkins-x/lighthouse/pkg/scmprovider"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,21 +20,8 @@ func TestMergeConfig(t *testing.T) {
 	owner := "myorg"
 	repo := "loadtest"
 	ref := "master"
-	fullName := scm.Join(owner, repo)
-
-	scmClient, fakeData := fake.NewDefault()
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-
-	fakeData.Repositories = []*scm.Repository{
-		{
-			Namespace: owner,
-			Name:      repo,
-			FullName:  fullName,
-			Branch:    "master",
-		},
-	}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
 	require.NoError(t, err, "failed to create filebrowsers")
 
 	cfg := &config.Config{}
@@ -55,18 +39,16 @@ func TestMergeConfig(t *testing.T) {
 }
 
 func TestInvalidConfigs(t *testing.T) {
-	scmClient, _ := fake.NewDefault()
-	scmProvider := scmprovider.ToClient(scmClient, "my-bot")
-
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
-	require.NoError(t, err, "failed to create filebrowsers")
-	fc := filebrowser.NewFetchCache()
-
 	invalidRepos := []string{"duplicate-presubmit", "duplicate-postsubmit"}
 	for _, repo := range invalidRepos {
 		owner := "myorg"
 		ref := "master"
-		_, err := LoadTriggerConfig(fileBrowsers, fc, NewResolverCache(), owner, repo, ref)
+
+		fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+		require.NoError(t, err, "failed to create filebrowsers")
+
+		fc := filebrowser.NewFetchCache()
+		_, err = LoadTriggerConfig(fileBrowsers, fc, NewResolverCache(), owner, repo, ref)
 		require.Errorf(t, err, "should have failed to load triggers from repo %s/%s with ref %s", owner, repo, ref)
 
 		t.Logf("got expected error loading invalid configuration on repo %s of: %s", repo, err.Error())
@@ -74,7 +56,7 @@ func TestInvalidConfigs(t *testing.T) {
 }
 
 func TestEmptyDirectoryDoesNotFail(t *testing.T) {
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data", "empty_dir")))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data", "empty_dir"), false))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
@@ -100,7 +82,7 @@ func TestLoadJobFromURL(t *testing.T) {
 
 	j := &job.Base{}
 	fc := filebrowser.NewFetchCache()
-	err := loadJobBaseFromSourcePath(nil, fc, NewResolverCache(), j, "", "", "https://raw.githubusercontent.com/rawlingsj/test/master/foo.yaml", "")
+	err := loadJobBaseFromSourcePath(nil, nil, fc, NewResolverCache(), j, "", "", "https://raw.githubusercontent.com/rawlingsj/test/master/foo.yaml", "")
 	assert.NoError(t, err, "should not have an error returned")
 	assert.Equal(t, "jenkinsxio/chuck:0.0.1", j.PipelineRunSpec.PipelineSpec.Tasks[0].TaskSpec.Steps[0].Image, "image name for task is not correct")
 }
diff --git a/pkg/webhook/create_agent_integration_test.go b/pkg/webhook/create_agent_integration_test.go
new file mode 100644
index 00000000..7e4fd8e4
--- /dev/null
+++ b/pkg/webhook/create_agent_integration_test.go
@@ -0,0 +1,89 @@
+package webhook
+
+import (
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/jenkins-x/go-scm/scm"
+	"github.com/jenkins-x/lighthouse/pkg/clients"
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
+	"github.com/jenkins-x/lighthouse/pkg/git"
+	gitv2 "github.com/jenkins-x/lighthouse/pkg/git/v2"
+	"github.com/jenkins-x/lighthouse/pkg/launcher"
+	"github.com/jenkins-x/lighthouse/pkg/plugins"
+	"github.com/jenkins-x/lighthouse/pkg/util"
+	"github.com/jenkins-x/lighthouse/pkg/watcher"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestCreateAgentIntegration(t *testing.T) {
+	owner := os.Getenv("GIT_OWNER")
+	repo := os.Getenv("GIT_REPO")
+	ref := os.Getenv("GIT_REF")
+	gitUser := os.Getenv("GIT_USERNAME")
+	gitToken := os.Getenv("GIT_TOKEN")
+	if owner == "" || repo == "" || ref == "" || gitUser == "" || gitToken == "" {
+		t.Skipf("skipping integration test as missing $GIT_OWNER/$GIT_REPO/$GIT_REF/$GIT_USERNAME/$GIT_TOKEN")
+		return
+	}
+
+	s := Server{}
+	s.ConfigAgent = &config.Agent{}
+	s.Plugins = &plugins.ConfigAgent{}
+
+	cfg := s.ConfigAgent.Config
+
+	_, err := watcher.SetupConfigMapWatchers("jx", s.ConfigAgent, s.Plugins)
+	assert.NoError(t, err)
+
+	_, scmClient, serverURL, _, err := util.GetSCMClient("", cfg)
+	assert.NoError(t, err)
+
+	_, kubeClient, lhClient, _, err := clients.GetAPIClients()
+	assert.NoError(t, err)
+
+	gitClient, err := git.NewClient(serverURL, util.GitKind(cfg))
+	assert.NoError(t, err)
+
+	configureOpts := func(opts *gitv2.ClientFactoryOpts) {
+		opts.Token = func() []byte {
+			return []byte(gitToken)
+
+		}
+		opts.GitUser = func() (name, email string, err error) {
+			name = gitUser
+			return
+		}
+		opts.Username = func() (login string, err error) {
+			login = gitUser
+			return
+		}
+
+	}
+
+	gitFactory, err := gitv2.NewClientFactory(configureOpts)
+	assert.NoError(t, err)
+
+	fb := filebrowser.NewFileBrowserFromGitClient(gitFactory)
+
+	s.FileBrowsers, err = filebrowser.NewFileBrowsers(serverURL, fb)
+	assert.NoError(t, err)
+
+	s.InRepoCache, err = lru.New(5000)
+	assert.NoError(t, err)
+
+	s.ClientAgent = &plugins.ClientAgent{
+		BotName:           "test-bot",
+		SCMProviderClient: scmClient,
+		KubernetesClient:  kubeClient,
+		GitClient:         gitClient,
+		LighthouseClient:  lhClient.LighthouseV1alpha1().LighthouseJobs("jx"),
+		LauncherClient:    launcher.NewLauncher(lhClient, "jx"),
+	}
+
+	l := logrus.WithField("Repository", scm.Join(owner, repo)+"@"+ref)
+	_, err = s.CreateAgent(l, owner, repo, ref)
+	assert.NoError(t, err)
+}