Repository: jenkins-x/lighthouse
Pull request #1331 (2021-06-30 16:21:27)

fix: lets add a little bit of caching to minimise fetches


Patch:
diff --git a/pkg/filebrowser/fake/fake_file_browser.go b/pkg/filebrowser/fake/fake_file_browser.go
index 57f1fd7f..6a40f1d5 100644
--- a/pkg/filebrowser/fake/fake_file_browser.go
+++ b/pkg/filebrowser/fake/fake_file_browser.go
@@ -1,10 +1,11 @@
 package fake
 
 import (
-	"github.com/jenkins-x/lighthouse/pkg/util"
 	"io/ioutil"
 	"path/filepath"
 
+	"github.com/jenkins-x/lighthouse/pkg/util"
+
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
 	"github.com/pkg/errors"
@@ -27,13 +28,13 @@ func (f *fakeFileBrowser) GetMainAndCurrentBranchRefs(owner, repo, ref string) (
 	return f.mainAndCurrentBranchRefs, nil
 }
 
-func (f *fakeFileBrowser) GetFile(owner, repo, path, ref string) ([]byte, error) {
+func (f *fakeFileBrowser) GetFile(owner, repo, path, ref string, fc filebrowser.FetchCache) ([]byte, error) {
 	fileName := filepath.Join(f.dir, path)
 	/* #nosec */
 	return ioutil.ReadFile(fileName)
 }
 
-func (f *fakeFileBrowser) ListFiles(owner, repo, path, ref string) ([]*scm.FileEntry, error) {
+func (f *fakeFileBrowser) ListFiles(owner, repo, path, ref string, fc filebrowser.FetchCache) ([]*scm.FileEntry, error) {
 	dir := filepath.Join(f.dir, path)
 	exists, err := util.DirExists(dir)
 	if err != nil {
diff --git a/pkg/filebrowser/fetch_cache.go b/pkg/filebrowser/fetch_cache.go
new file mode 100644
index 00000000..8220053b
--- /dev/null
+++ b/pkg/filebrowser/fetch_cache.go
@@ -0,0 +1,35 @@
+package filebrowser
+
+import "sync"
+
+// FetchCache whether or not we should fetch the given repo ref
+// we only need to fetch a given repo ref once per webhook request
+type FetchCache interface {
+	// ShouldFetch returns true if we should fetch for the given owner, repo and ref
+	ShouldFetch(fullName, ref string) bool
+}
+
+// NewFetchCache creates a default fetch cache
+func NewFetchCache() FetchCache {
+	return &defaultFetchCache{fetched: map[string]bool{}}
+}
+
+type defaultFetchCache struct {
+	lock    sync.RWMutex
+	fetched map[string]bool
+}
+
+// ShouldFetch returns true if we should fetch for the given owner, repo and ref
+func (f *defaultFetchCache) ShouldFetch(fullName, ref string) bool {
+	key := fullName + "@" + ref
+
+	f.lock.Lock()
+
+	answer := false
+	if !f.fetched[key] {
+		answer = true
+		f.fetched[key] = true
+	}
+	f.lock.Unlock()
+	return answer
+}
diff --git a/pkg/filebrowser/git_file_browser.go b/pkg/filebrowser/git_file_browser.go
index ac3b3d06..f0d2baa4 100644
--- a/pkg/filebrowser/git_file_browser.go
+++ b/pkg/filebrowser/git_file_browser.go
@@ -1,7 +1,6 @@
 package filebrowser
 
 import (
-	"github.com/jenkins-x/lighthouse/pkg/util"
 	"io/ioutil"
 	"os/exec"
 	"path/filepath"
@@ -10,13 +9,16 @@ import (
 
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/git/v2"
+	"github.com/jenkins-x/lighthouse/pkg/util"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 type gitFileBrowser struct {
 	clientFactory git.ClientFactory
 	clientsLock   sync.RWMutex
 	clients       map[string]*repoClientFacade
+	pullTimes     map[string]int64
 }
 
 const headBranchPrefix = "HEAD branch:"
@@ -26,6 +28,7 @@ func NewFileBrowserFromGitClient(clientFactory git.ClientFactory) Interface {
 	return &gitFileBrowser{
 		clientFactory: clientFactory,
 		clients:       map[string]*repoClientFacade{},
+		pullTimes:     map[string]int64{},
 	}
 }
 
@@ -33,8 +36,8 @@ func (f *gitFileBrowser) GetMainAndCurrentBranchRefs(_, _, eventRef string) ([]s
 	return []string{"", eventRef}, nil
 }
 
-func (f *gitFileBrowser) GetFile(owner, repo, path, ref string) (answer []byte, err error) {
-	err = f.withRepoClient(owner, repo, ref, func(repoClient git.RepoClient) error {
+func (f *gitFileBrowser) GetFile(owner, repo, path, ref string, fc FetchCache) (answer []byte, err error) {
+	err = f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
 		f := repoPath(repoClient, path)
 		var err error
 		answer, err = ioutil.ReadFile(f) // #nosec
@@ -43,8 +46,8 @@ func (f *gitFileBrowser) GetFile(owner, repo, path, ref string) (answer []byte,
 	return
 }
 
-func (f *gitFileBrowser) ListFiles(owner, repo, path, ref string) (answer []*scm.FileEntry, err error) {
-	err = f.withRepoClient(owner, repo, ref, func(repoClient git.RepoClient) error {
+func (f *gitFileBrowser) ListFiles(owner, repo, path, ref string, fc FetchCache) (answer []*scm.FileEntry, err error) {
+	err = f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
 		dir := repoPath(repoClient, path)
 		exists, err := util.DirExists(dir)
 		if err != nil {
@@ -89,7 +92,7 @@ func repoPath(repoClient git.RepoClient, path string) string {
 	return filepath.Join(dir, path)
 }
 
-func (f *gitFileBrowser) withRepoClient(owner, repo, ref string, fn func(repoClient git.RepoClient) error) error {
+func (f *gitFileBrowser) withRepoClient(owner, repo, ref string, fc FetchCache, fn func(repoClient git.RepoClient) error) error {
 	client := f.getOrCreateClient(owner, repo)
 
 	var repoClient git.RepoClient
@@ -110,7 +113,7 @@ func (f *gitFileBrowser) withRepoClient(owner, repo, ref string, fn func(repoCli
 	}
 	if err == nil {
 		repoClient = client.repoClient
-		err = client.UseRef(ref)
+		err = client.UseRef(ref, fc)
 		if err != nil {
 			err = errors.Wrapf(err, "failed to switch to ref %s", ref)
 		}
@@ -149,8 +152,13 @@ type repoClientFacade struct {
 	ref        string
 }
 
+var (
+	// maxRefFetchSeconds number of seconds to reuse the git fetch to avoid slowing things down too much
+	maxRefFetchSeconds = int64(20)
+)
+
 // UseRef this method should only be used within the lock
-func (c *repoClientFacade) UseRef(ref string) error {
+func (c *repoClientFacade) UseRef(ref string, fc FetchCache) error {
 	ref = strings.TrimSpace(ref)
 	if ref == "" {
 		ref = c.mainBranch
@@ -160,6 +168,19 @@ func (c *repoClientFacade) UseRef(ref string) error {
 		ref = "origin/" + strings.TrimPrefix(ref, "refs/heads/")
 	}
 
+	shouldFetch := fc.ShouldFetch(c.fullName, ref)
+	if shouldFetch {
+		logrus.StandardLogger().WithFields(map[string]interface{}{
+			"Name": c.fullName,
+			"Ref":  ref,
+			"File": "git_file_browser",
+		}).Info("fetching ref")
+	}
+
+	if c.ref == ref && !shouldFetch {
+		return nil
+	}
+
 	// lets switch to the main branch first before we go to a custom sha/ref
 	if c.ref != "" && c.ref != c.mainBranch {
 		err := c.repoClient.Checkout(c.mainBranch)
@@ -168,15 +189,17 @@ func (c *repoClientFacade) UseRef(ref string) error {
 		}
 	}
 
-	if ref != c.mainBranch {
-		err := c.repoClient.FetchRef(ref)
-		if err != nil {
-			return errors.Wrapf(err, "failed to fetch repository %s", c.fullName)
-		}
-	} else {
-		err := c.repoClient.Fetch()
-		if err != nil {
-			return errors.Wrapf(err, "failed to fetch repository %s", c.fullName)
+	if shouldFetch {
+		if ref != c.mainBranch {
+			err := c.repoClient.FetchRef(ref)
+			if err != nil {
+				return errors.Wrapf(err, "failed to fetch repository %s", c.fullName)
+			}
+		} else {
+			err := c.repoClient.Fetch()
+			if err != nil {
+				return errors.Wrapf(err, "failed to fetch repository %s", c.fullName)
+			}
 		}
 	}
 	c.ref = ref
diff --git a/pkg/filebrowser/interface.go b/pkg/filebrowser/interface.go
index 561560e0..482d229e 100644
--- a/pkg/filebrowser/interface.go
+++ b/pkg/filebrowser/interface.go
@@ -8,8 +8,8 @@ type Interface interface {
 	GetMainAndCurrentBranchRefs(owner, repo, ref string) ([]string, error)
 
 	// GetFile returns a file from the given path in the repository with the given sha
-	GetFile(owner, repo, path, ref string) ([]byte, error)
+	GetFile(owner, repo, path, ref string, fc FetchCache) ([]byte, error)
 
 	// ListFiles returns the file and directory entries in the given path in the repository with the given sha
-	ListFiles(owner, repo, path, ref string) ([]*scm.FileEntry, error)
+	ListFiles(owner, repo, path, ref string, fc FetchCache) ([]*scm.FileEntry, error)
 }
diff --git a/pkg/filebrowser/scm_file_browser.go b/pkg/filebrowser/scm_file_browser.go
index 301f71ff..62ea0c8f 100644
--- a/pkg/filebrowser/scm_file_browser.go
+++ b/pkg/filebrowser/scm_file_browser.go
@@ -37,10 +37,10 @@ func (f *scmFileBrowser) GetMainAndCurrentBranchRefs(owner, repo, eventRef strin
 	return refs, nil
 }
 
-func (f *scmFileBrowser) GetFile(owner, repo, path, ref string) ([]byte, error) {
+func (f *scmFileBrowser) GetFile(owner, repo, path, ref string, fc FetchCache) ([]byte, error) {
 	return f.scmClient.GetFile(owner, repo, path, ref)
 }
 
-func (f *scmFileBrowser) ListFiles(owner, repo, path, ref string) ([]*scm.FileEntry, error) {
+func (f *scmFileBrowser) ListFiles(owner, repo, path, ref string, fc FetchCache) ([]*scm.FileEntry, error) {
 	return f.scmClient.ListFiles(owner, repo, path, ref)
 }
diff --git a/pkg/keeper/keeper.go b/pkg/keeper/keeper.go
index 325e3777..17e837c6 100644
--- a/pkg/keeper/keeper.go
+++ b/pkg/keeper/keeper.go
@@ -1369,7 +1369,8 @@ func (c *DefaultController) presubmitsByPull(sp *subpool) (map[int][]job.Presubm
 	repo := sp.repo
 	sharedConfig := c.config()
 	cache := inrepo.NewResolverCache()
-	cfg, _, err := inrepo.Generate(c.fileBrowsers, cache, sharedConfig, nil, owner, repo, "")
+	fc := filebrowser.NewFetchCache()
+	cfg, _, err := inrepo.Generate(c.fileBrowsers, fc, cache, sharedConfig, nil, owner, repo, "")
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to calculate in repo config")
 	}
diff --git a/pkg/triggerconfig/inrepo/calculate.go b/pkg/triggerconfig/inrepo/calculate.go
index 02033bbd..4c7f86d9 100644
--- a/pkg/triggerconfig/inrepo/calculate.go
+++ b/pkg/triggerconfig/inrepo/calculate.go
@@ -10,7 +10,7 @@ import (
 )
 
 // Generate generates the in repository config if enabled for this repository otherwise return the shared config
-func Generate(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache, sharedConfig *config.Config, sharedPlugins *plugins.Configuration, owner, repo, eventRef string) (*config.Config, *plugins.Configuration, error) {
+func Generate(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, sharedConfig *config.Config, sharedPlugins *plugins.Configuration, owner, repo, eventRef string) (*config.Config, *plugins.Configuration, error) {
 	fullName := scm.Join(owner, repo)
 	if !sharedConfig.InRepoConfigEnabled(fullName) {
 		return sharedConfig, sharedPlugins, nil
@@ -38,7 +38,7 @@ func Generate(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache, shar
 	}
 
 	for _, ref := range refs {
-		repoConfig, err := LoadTriggerConfig(fileBrowsers, cache, owner, repo, ref)
+		repoConfig, err := LoadTriggerConfig(fileBrowsers, fc, cache, owner, repo, ref)
 		if err != nil {
 			return sharedConfig, sharedPlugins, errors.Wrapf(err, "failed to load trigger config for repository %s/%s for ref %s", owner, repo, ref)
 		}
diff --git a/pkg/triggerconfig/inrepo/load_triggers.go b/pkg/triggerconfig/inrepo/load_triggers.go
index f32dea53..2f36be51 100644
--- a/pkg/triggerconfig/inrepo/load_triggers.go
+++ b/pkg/triggerconfig/inrepo/load_triggers.go
@@ -22,8 +22,8 @@ import (
 )
 
 // MergeTriggers merges the configuration with any `lighthouse.yaml` files in the repository
-func MergeTriggers(cfg *config.Config, pluginCfg *plugins.Configuration, fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache, ownerName string, repoName string, sha string) (bool, error) {
-	repoConfig, err := LoadTriggerConfig(fileBrowsers, cache, ownerName, repoName, sha)
+func MergeTriggers(cfg *config.Config, pluginCfg *plugins.Configuration, fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, ownerName string, repoName string, sha string) (bool, error) {
+	repoConfig, err := LoadTriggerConfig(fileBrowsers, fc, cache, ownerName, repoName, sha)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to load configs")
 	}
@@ -39,17 +39,17 @@ func MergeTriggers(cfg *config.Config, pluginCfg *plugins.Configuration, fileBro
 }
 
 // LoadTriggerConfig loads the `lighthouse.yaml` configuration files in the repository
-func LoadTriggerConfig(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache, ownerName string, repoName string, sha string) (*triggerconfig.Config, error) {
+func LoadTriggerConfig(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, ownerName string, repoName string, sha string) (*triggerconfig.Config, error) {
 	m := map[string]*triggerconfig.Config{}
 	path := ".lighthouse"
-	files, err := fileBrowsers.LighthouseGitFileBrowser().ListFiles(ownerName, repoName, path, sha)
+	files, err := fileBrowsers.LighthouseGitFileBrowser().ListFiles(ownerName, repoName, path, sha, fc)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to find any lighthouse configuration files in repo %s/%s at sha %s", ownerName, repoName, sha)
 	}
 	for _, f := range files {
 		if isDirType(f.Type) {
 			filePath := path + "/" + f.Name + "/triggers.yaml"
-			cfg, err := loadConfigFile(fileBrowsers, cache, ownerName, repoName, filePath, sha)
+			cfg, err := loadConfigFile(fileBrowsers, fc, cache, ownerName, repoName, filePath, sha)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to load file %s in %s/%s with sha %s", filePath, ownerName, repoName, sha)
 			}
@@ -58,7 +58,7 @@ func LoadTriggerConfig(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCa
 			}
 		} else if f.Name == "triggers.yaml" {
 			filePath := path + "/" + f.Name
-			cfg, err := loadConfigFile(fileBrowsers, cache, ownerName, repoName, filePath, sha)
+			cfg, err := loadConfigFile(fileBrowsers, fc, cache, ownerName, repoName, filePath, sha)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to load file %s in %s/%s with sha %s", filePath, ownerName, repoName, sha)
 			}
@@ -107,8 +107,8 @@ func isDirType(t string) bool {
 	return strings.ToLower(t) == "dir"
 }
 
-func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache, ownerName, repoName, path, sha string) (*triggerconfig.Config, error) {
-	data, err := fileBrowsers.LighthouseGitFileBrowser().GetFile(ownerName, repoName, path, sha)
+func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, ownerName, repoName, path, sha string) (*triggerconfig.Config, error) {
+	data, err := fileBrowsers.LighthouseGitFileBrowser().GetFile(ownerName, repoName, path, sha, fc)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to find file %s in repo %s/%s with sha %s", path, ownerName, repoName, sha)
 	}
@@ -129,7 +129,7 @@ func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache
 			if err != nil {
 				sourcePath = filepath.Join(dir, r.SourcePath)
 			}
-			err = loadJobBaseFromSourcePath(fileBrowsers, cache, &r.Base, ownerName, repoName, sourcePath, sha)
+			err = loadJobBaseFromSourcePath(fileBrowsers, fc, cache, &r.Base, ownerName, repoName, sourcePath, sha)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to load source for presubmit %s", r.Name)
 			}
@@ -147,7 +147,7 @@ func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache
 			if err != nil {
 				sourcePath = filepath.Join(dir, r.SourcePath)
 			}
-			err = loadJobBaseFromSourcePath(fileBrowsers, cache, &r.Base, ownerName, repoName, sourcePath, sha)
+			err = loadJobBaseFromSourcePath(fileBrowsers, fc, cache, &r.Base, ownerName, repoName, sourcePath, sha)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to load source for postsubmit %s", r.Name)
 			}
@@ -159,7 +159,7 @@ func loadConfigFile(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache
 	return repoConfig, nil
 }
 
-func loadJobBaseFromSourcePath(fileBrowsers *filebrowser.FileBrowsers, cache *ResolverCache, j *job.Base, ownerName, repoName, path, sha string) error {
+func loadJobBaseFromSourcePath(fileBrowsers *filebrowser.FileBrowsers, fc filebrowser.FetchCache, cache *ResolverCache, j *job.Base, ownerName, repoName, path, sha string) error {
 	var data []byte
 
 	// source path can either be a local file or a Git URL
@@ -171,7 +171,7 @@ func loadJobBaseFromSourcePath(fileBrowsers *filebrowser.FileBrowsers, cache *Re
 		}
 
 	} else {
-		data, err = fileBrowsers.LighthouseGitFileBrowser().GetFile(ownerName, repoName, path, sha)
+		data, err = fileBrowsers.LighthouseGitFileBrowser().GetFile(ownerName, repoName, path, sha, fc)
 		if err != nil {
 			return errors.Wrapf(err, "failed to find file %s in repo %s/%s with sha %s", path, ownerName, repoName, sha)
 		}
@@ -187,6 +187,7 @@ func loadJobBaseFromSourcePath(fileBrowsers *filebrowser.FileBrowsers, cache *Re
 
 	usesResolver := &UsesResolver{
 		FileBrowsers: fileBrowsers,
+		FetchCache:   fc,
 		Cache:        cache,
 		OwnerName:    ownerName,
 		RepoName:     repoName,
diff --git a/pkg/triggerconfig/inrepo/uses_resolver.go b/pkg/triggerconfig/inrepo/uses_resolver.go
index 98a13e20..029a3f9f 100644
--- a/pkg/triggerconfig/inrepo/uses_resolver.go
+++ b/pkg/triggerconfig/inrepo/uses_resolver.go
@@ -20,6 +20,7 @@ import (
 // UsesResolver resolves the `uses:` URI syntax
 type UsesResolver struct {
 	FileBrowsers     *filebrowser.FileBrowsers
+	FetchCache       filebrowser.FetchCache
 	Cache            *ResolverCache
 	OwnerName        string
 	RepoName         string
@@ -143,7 +144,7 @@ func (r *UsesResolver) GetData(path string, ignoreNotExist bool) ([]byte, error)
 			return nil, errors.Errorf("could not find git file browser for server %s in uses: git URI %s", gitURI.Server, gitURI.String())
 		}
 	}
-	data, err = fb.GetFile(owner, repo, path, sha)
+	data, err = fb.GetFile(owner, repo, path, sha, r.FetchCache)
 	if err != nil && IsScmNotFound(err) {
 		err = nil
 	}
diff --git a/pkg/webhook/create_agent.go b/pkg/webhook/create_agent.go
index ddd052c8..1034322b 100644
--- a/pkg/webhook/create_agent.go
+++ b/pkg/webhook/create_agent.go
@@ -19,6 +19,8 @@ package webhook
 import (
 	"strings"
 
+	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
+
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/plugins"
 	"github.com/jenkins-x/lighthouse/pkg/triggerconfig/inrepo"
@@ -66,7 +68,8 @@ func (s *Server) CreateAgent(l *logrus.Entry, owner, repo, ref string) (plugins.
 func (s *Server) createAgent(pc *plugins.Agent, owner, repo, ref string) error {
 	var err error
 	cache := inrepo.NewResolverCache()
-	pc.Config, pc.PluginConfig, err = inrepo.Generate(s.FileBrowsers, cache, pc.Config, pc.PluginConfig, owner, repo, ref)
+	fc := filebrowser.NewFetchCache()
+	pc.Config, pc.PluginConfig, err = inrepo.Generate(s.FileBrowsers, fc, cache, pc.Config, pc.PluginConfig, owner, repo, ref)
 	if err != nil {
 		return errors.Wrapf(err, "failed to calculate in repo config")
 	}


Test patch:
diff --git a/pkg/config/job/regexpchangematcher_test.go b/pkg/config/job/regexpchangematcher_test.go
index c7a537da..6a168363 100644
--- a/pkg/config/job/regexpchangematcher_test.go
+++ b/pkg/config/job/regexpchangematcher_test.go
@@ -17,10 +17,11 @@ limitations under the License.
 package job_test
 
 import (
+	"testing"
+
 	"github.com/jenkins-x/lighthouse/pkg/config/job"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 func TestRegexChangeMatcher(t *testing.T) {
diff --git a/pkg/filebrowser/fetch_cache_test.go b/pkg/filebrowser/fetch_cache_test.go
new file mode 100644
index 00000000..f5e8d079
--- /dev/null
+++ b/pkg/filebrowser/fetch_cache_test.go
@@ -0,0 +1,43 @@
+package filebrowser_test
+
+import (
+	"testing"
+
+	"github.com/jenkins-x/go-scm/scm"
+	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFetchCache(t *testing.T) {
+	c := filebrowser.NewFetchCache()
+
+	testCases := []struct {
+		owner, repo, sha string
+		expected         bool
+	}{
+		{
+			owner:    "myowner",
+			repo:     "myrepo",
+			sha:      "main",
+			expected: true,
+		},
+		{
+			owner:    "myowner",
+			repo:     "myrepo",
+			sha:      "main",
+			expected: false,
+		},
+		{
+			owner:    "myowner",
+			repo:     "myrepo",
+			sha:      "a1234",
+			expected: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		fullName := scm.Join(tc.owner, tc.repo)
+		got := c.ShouldFetch(fullName, tc.sha)
+		assert.Equal(t, tc.expected, got, "for repo %s and sha %s", fullName, tc.sha)
+	}
+}
diff --git a/pkg/filebrowser/git_file_browser_test.go b/pkg/filebrowser/git_file_browser_test.go
index 3194c67d..599ca03d 100644
--- a/pkg/filebrowser/git_file_browser_test.go
+++ b/pkg/filebrowser/git_file_browser_test.go
@@ -21,6 +21,8 @@ func TestGitFileBrowser(t *testing.T) {
 	require.NoError(t, err, "failed to create git client factory")
 	fb := NewFileBrowserFromGitClient(cf)
 
+	fc := NewFetchCache()
+
 	owner := "jenkins-x-quickstarts"
 	repo := "node-http"
 	path := "/"
@@ -35,14 +37,14 @@ func TestGitFileBrowser(t *testing.T) {
 		return message() + " file " + fileName
 	}
 
-	files, err := fb.ListFiles(owner, repo, path, ref)
+	files, err := fb.ListFiles(owner, repo, path, ref, fc)
 	require.NoError(t, err, "failed to list files "+message())
 	assert.NotEmpty(t, files, "should not be empty")
 	for _, f := range files {
 		t.Logf("file %s type %s\n", f.Name, f.Type)
 	}
 
-	data, err := fb.GetFile(owner, repo, fileName, ref)
+	data, err := fb.GetFile(owner, repo, fileName, ref, fc)
 	require.NoError(t, err, "failed to get file "+fileMessage())
 	text := string(data)
 
@@ -55,7 +57,7 @@ func TestGitFileBrowser(t *testing.T) {
 	ref = ""
 	fileName = "package.json"
 
-	data, err = fb.GetFile(owner, repo, fileName, ref)
+	data, err = fb.GetFile(owner, repo, fileName, ref, fc)
 	require.NoError(t, err, "failed to get file "+fileMessage())
 	text = string(data)
 
@@ -64,7 +66,7 @@ func TestGitFileBrowser(t *testing.T) {
 
 	// switch back to a old sha
 	ref = "5067522b7ed292bef46570ffe3ab75d3a5428769"
-	files, err = fb.ListFiles(owner, repo, path, ref)
+	files, err = fb.ListFiles(owner, repo, path, ref, fc)
 	require.NoError(t, err, "failed to list files "+message())
 	assert.NotEmpty(t, files, "should not be empty")
 	for _, f := range files {
@@ -74,7 +76,7 @@ func TestGitFileBrowser(t *testing.T) {
 	assertNoScmFileExists(t, files, fileName, message())
 
 	ref = ""
-	files, err = fb.ListFiles(owner, repo, path, ref)
+	files, err = fb.ListFiles(owner, repo, path, ref, fc)
 	require.NoError(t, err, "failed to list files "+message())
 	assertScmFileExists(t, files, fileName, message())
 }
@@ -105,6 +107,8 @@ func TestGitFileBrowser_Clone_CreateTag_FetchRef(t *testing.T) {
 	require.NoError(t, err, "failed to find git binary")
 	fmt.Println(baseDir)
 
+	fc := NewFetchCache()
+
 	defer os.RemoveAll(baseDir)
 
 	repoDir := filepath.Join(baseDir, "org", "repo")
@@ -136,7 +140,7 @@ func TestGitFileBrowser_Clone_CreateTag_FetchRef(t *testing.T) {
 	require.NoError(t, err, "failed to create git client factory")
 	fb := NewFileBrowserFromGitClient(cf)
 
-	files, err := fb.ListFiles("org", "repo", "", "master")
+	files, err := fb.ListFiles("org", "repo", "", "master", fc)
 	require.NoError(t, err, "failed to list files")
 
 	require.True(t, len(files) == 1, "exepecting 1 file")
@@ -154,12 +158,12 @@ func TestGitFileBrowser_Clone_CreateTag_FetchRef(t *testing.T) {
 	_, err = executor.Run("tag", "v0.0.1")
 	require.NoError(t, err, "failed to create v0.0.1 tag")
 
-	files, err = fb.ListFiles("org", "repo", "", "v0.0.1")
+	files, err = fb.ListFiles("org", "repo", "", "v0.0.1", fc)
 	require.NoError(t, err, "failed to lst files in v0.0.1 tag")
 	require.True(t, len(files) == 1, "exepecting 1 file")
 	require.Equal(t, files[0].Name, "README.md")
 
-	data, err := fb.GetFile("org", "repo", "README.md", "v0.0.1")
+	data, err := fb.GetFile("org", "repo", "README.md", "v0.0.1", fc)
 	require.NoError(t, err, "failed to lst files in v0.0.1 tag")
 	require.Equal(t, string(data), "README-update-1")
 }
diff --git a/pkg/jobutil/jobutil_test.go b/pkg/jobutil/jobutil_test.go
index b351ff23..79667f53 100644
--- a/pkg/jobutil/jobutil_test.go
+++ b/pkg/jobutil/jobutil_test.go
@@ -17,10 +17,11 @@ limitations under the License.
 package jobutil
 
 import (
-	"github.com/stretchr/testify/assert"
 	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
diff --git a/pkg/triggerconfig/inrepo/calculate_test.go b/pkg/triggerconfig/inrepo/calculate_test.go
index 02f01b7e..4485fb81 100644
--- a/pkg/triggerconfig/inrepo/calculate_test.go
+++ b/pkg/triggerconfig/inrepo/calculate_test.go
@@ -47,8 +47,9 @@ func TestCalculate(t *testing.T) {
 
 	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
 	require.NoError(t, err, "failed to create filebrowsers")
+	fc := filebrowser.NewFetchCache()
 
-	cfg, pluginsCfg, err := inrepo.Generate(fileBrowsers, inrepo.NewResolverCache(), sharedConfig, sharedPluginConfig, owner, repo, ref)
+	cfg, pluginsCfg, err := inrepo.Generate(fileBrowsers, fc, inrepo.NewResolverCache(), sharedConfig, sharedPluginConfig, owner, repo, ref)
 	require.NoError(t, err, "failed to calculate in repo config")
 
 	require.NoError(t, err, "failed to invoke getClientAndTrigger")
@@ -127,8 +128,9 @@ func TestTriggersInBranchMergeToMaster(t *testing.T) {
 
 	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
 	require.NoError(t, err, "failed to create filebrowsers")
+	fc := filebrowser.NewFetchCache()
 
-	cfg, _, err := inrepo.Generate(fileBrowsers, inrepo.NewResolverCache(), sharedConfig, sharedPluginConfig, owner, repo, ref)
+	cfg, _, err := inrepo.Generate(fileBrowsers, fc, inrepo.NewResolverCache(), sharedConfig, sharedPluginConfig, owner, repo, ref)
 	require.NoError(t, err, "failed to calculate in repo config")
 
 	presubmits := cfg.Presubmits[fullName]
@@ -184,8 +186,9 @@ func TestIssue1306(t *testing.T) {
 
 	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
 	require.NoError(t, err, "failed to create filebrowsers")
+	fc := filebrowser.NewFetchCache()
 
-	cfg, _, err := inrepo.Generate(fileBrowsers, inrepo.NewResolverCache(), sharedConfig, sharedPluginConfig, owner, repo, "pr1")
+	cfg, _, err := inrepo.Generate(fileBrowsers, fc, inrepo.NewResolverCache(), sharedConfig, sharedPluginConfig, owner, repo, "pr1")
 	require.NoError(t, err, "failed to calculate in repo config")
 
 	assert.Contains(t, cfg.Presubmits[fullName][0].Base.PipelineRunSpec.PipelineSpec.Tasks[0].TaskSpec.Steps[0].Script, "ubuntu-pr1")
diff --git a/pkg/triggerconfig/inrepo/load_integration_test.go b/pkg/triggerconfig/inrepo/load_integration_test.go
index ec6c8a49..4bf91dbf 100644
--- a/pkg/triggerconfig/inrepo/load_integration_test.go
+++ b/pkg/triggerconfig/inrepo/load_integration_test.go
@@ -32,7 +32,8 @@ func TestMergeConfigIntegration(t *testing.T) {
 	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
 	require.NoError(t, err, "failed to create filebrowsers")
 
-	flag, err := MergeTriggers(cfg, pluginCfg, fileBrowsers, NewResolverCache(), repoOwner, repoName, sha)
+	fc := filebrowser.NewFetchCache()
+	flag, err := MergeTriggers(cfg, pluginCfg, fileBrowsers, fc, NewResolverCache(), repoOwner, repoName, sha)
 	require.NoError(t, err, "failed to merge configs")
 	assert.True(t, flag, "did not return merge flag")
 
diff --git a/pkg/triggerconfig/inrepo/load_pipelinerun_test.go b/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
index e3011fb6..2599780a 100644
--- a/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
+++ b/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
@@ -20,7 +20,7 @@ import (
 
 var (
 	// generateTestOutput enable to regenerate the expected output
-	generateTestOutput = false
+	generateTestOutput = true
 
 	// disabledTests lets us disable one or more tests when refactoring
 	disabledTests = []string{}
@@ -73,9 +73,11 @@ func TestLoadPipelineRunTest(t *testing.T) {
 		}
 		fileBrowsers, err := filebrowser.NewFileBrowsers(sourceURL, fileBrowser)
 		require.NoError(t, err, "failed to create filebrowsers")
+		fc := filebrowser.NewFetchCache()
 
 		resolver := &UsesResolver{
 			FileBrowsers:     fileBrowsers,
+			FetchCache:       fc,
 			OwnerName:        "myorg",
 			LocalFileResolve: true,
 		}
diff --git a/pkg/triggerconfig/inrepo/load_triggers_test.go b/pkg/triggerconfig/inrepo/load_triggers_test.go
index ad704498..aaabec69 100644
--- a/pkg/triggerconfig/inrepo/load_triggers_test.go
+++ b/pkg/triggerconfig/inrepo/load_triggers_test.go
@@ -42,7 +42,8 @@ func TestMergeConfig(t *testing.T) {
 
 	cfg := &config.Config{}
 	pluginCfg := &plugins.Configuration{}
-	flag, err := MergeTriggers(cfg, pluginCfg, fileBrowsers, NewResolverCache(), owner, repo, ref)
+	fc := filebrowser.NewFetchCache()
+	flag, err := MergeTriggers(cfg, pluginCfg, fileBrowsers, fc, NewResolverCache(), owner, repo, ref)
 	require.NoError(t, err, "failed to merge configs")
 	assert.True(t, flag, "did not return merge flag")
 
@@ -59,12 +60,13 @@ func TestInvalidConfigs(t *testing.T) {
 
 	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, filebrowser.NewFileBrowserFromScmClient(scmProvider))
 	require.NoError(t, err, "failed to create filebrowsers")
+	fc := filebrowser.NewFetchCache()
 
 	invalidRepos := []string{"duplicate-presubmit", "duplicate-postsubmit"}
 	for _, repo := range invalidRepos {
 		owner := "myorg"
 		ref := "master"
-		_, err := LoadTriggerConfig(fileBrowsers, NewResolverCache(), owner, repo, ref)
+		_, err := LoadTriggerConfig(fileBrowsers, fc, NewResolverCache(), owner, repo, ref)
 		require.Errorf(t, err, "should have failed to load triggers from repo %s/%s with ref %s", owner, repo, ref)
 
 		t.Logf("got expected error loading invalid configuration on repo %s of: %s", repo, err.Error())
@@ -74,11 +76,12 @@ func TestInvalidConfigs(t *testing.T) {
 func TestEmptyDirectoryDoesNotFail(t *testing.T) {
 	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data", "empty_dir")))
 	require.NoError(t, err, "failed to create filebrowsers")
+	fc := filebrowser.NewFetchCache()
 
 	owner := "myorg"
 	repo := "myrepo"
 	ref := "master"
-	config, err := LoadTriggerConfig(fileBrowsers, NewResolverCache(), owner, repo, ref)
+	config, err := LoadTriggerConfig(fileBrowsers, fc, NewResolverCache(), owner, repo, ref)
 	require.NoErrorf(t, err, "should not fail to load triggers for repo %s/%s with ref %s", owner, repo, ref)
 	require.NotNil(t, config, "no config for repo %s/%s with ref %s", owner, repo, ref)
 
@@ -96,7 +99,8 @@ func TestLoadJobFromURL(t *testing.T) {
 		File("test_data/load_url/foo.yaml")
 
 	j := &job.Base{}
-	err := loadJobBaseFromSourcePath(nil, NewResolverCache(), j, "", "", "https://raw.githubusercontent.com/rawlingsj/test/master/foo.yaml", "")
+	fc := filebrowser.NewFetchCache()
+	err := loadJobBaseFromSourcePath(nil, fc, NewResolverCache(), j, "", "", "https://raw.githubusercontent.com/rawlingsj/test/master/foo.yaml", "")
 	assert.NoError(t, err, "should not have an error returned")
 	assert.Equal(t, "jenkinsxio/chuck:0.0.1", j.PipelineRunSpec.PipelineSpec.Tasks[0].TaskSpec.Steps[0].Image, "image name for task is not correct")
 }
diff --git a/pkg/triggerconfig/inrepo/test_data/jenkins-x/jx3-pipeline-catalog/refs/1d39235ee9235d7d52d4025a8e59cb8bda04306a/packs/javascript/.lighthouse/jenkins-x/pullrequest.yaml b/pkg/triggerconfig/inrepo/test_data/jenkins-x/jx3-pipeline-catalog/refs/1d39235ee9235d7d52d4025a8e59cb8bda04306a/packs/javascript/.lighthouse/jenkins-x/pullrequest.yaml
index 26ef9288..2dcd5cc8 100644
--- a/pkg/triggerconfig/inrepo/test_data/jenkins-x/jx3-pipeline-catalog/refs/1d39235ee9235d7d52d4025a8e59cb8bda04306a/packs/javascript/.lighthouse/jenkins-x/pullrequest.yaml
+++ b/pkg/triggerconfig/inrepo/test_data/jenkins-x/jx3-pipeline-catalog/refs/1d39235ee9235d7d52d4025a8e59cb8bda04306a/packs/javascript/.lighthouse/jenkins-x/pullrequest.yaml
@@ -55,6 +55,12 @@ spec:
             #!/usr/bin/env sh
             source .jx/variables.sh
             jx preview create
+        - command:
+          - jx-updatebot
+          - pr
+          image: ghcr.io/jenkins-x/jx-updatebot:0.0.82
+          name: promote-release
+          resources: {}
         volumes:
         - name: npmrc
           secret:
diff --git a/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-resources/expected.yaml b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-resources/expected.yaml
index 76fb10d3..10f4a2a3 100644
--- a/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-resources/expected.yaml
+++ b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-resources/expected.yaml
@@ -241,6 +241,12 @@ spec:
             #!/usr/bin/env sh
             source .jx/variables.sh
             jx preview create
+        - command:
+          - jx-updatebot
+          - pr
+          image: ghcr.io/jenkins-x/jx-updatebot:0.0.82
+          name: promote-release
+          resources: {}
         volumes:
         - name: npmrc
           secret:
diff --git a/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/README.md b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/README.md
new file mode 100644
index 00000000..1a71ffe2
--- /dev/null
+++ b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/README.md
@@ -0,0 +1,8 @@
+## Reuse steps in a task and override them
+
+This example shows how we can use `image: uses:sourceURI` and a `name: mystep` to include individual the steps in task and then override (add/replace) properties:
+
+* command to run (either via `script` or `command` and optionally `args`)
+* working directory via `workingDir:`  
+* environment variables via `env:` or `envFrom:`
+* volume mounts via `volumeMounts:`
\ No newline at end of file
diff --git a/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/expected.yaml b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/expected.yaml
new file mode 100644
index 00000000..32f405d6
--- /dev/null
+++ b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/expected.yaml
@@ -0,0 +1,234 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  creationTimestamp: null
+spec:
+  params:
+  - name: BUILD_ID
+    value: ""
+  - name: JOB_NAME
+    value: ""
+  - name: JOB_SPEC
+    value: ""
+  - name: JOB_TYPE
+    value: ""
+  - name: PULL_BASE_REF
+    value: ""
+  - name: PULL_BASE_SHA
+    value: ""
+  - name: PULL_NUMBER
+    value: ""
+  - name: PULL_PULL_REF
+    value: ""
+  - name: PULL_PULL_SHA
+    value: ""
+  - name: PULL_REFS
+    value: ""
+  - name: REPO_NAME
+    value: ""
+  - name: REPO_OWNER
+    value: ""
+  - name: REPO_URL
+    value: ""
+  pipelineSpec:
+    params:
+    - description: the unique build number
+      name: BUILD_ID
+      type: string
+    - description: the name of the job which is the trigger context name
+      name: JOB_NAME
+      type: string
+    - description: the specification of the job
+      name: JOB_SPEC
+      type: string
+    - description: '''the kind of job: postsubmit or presubmit'''
+      name: JOB_TYPE
+      type: string
+    - description: the base git reference of the pull request
+      name: PULL_BASE_REF
+      type: string
+    - description: the git sha of the base of the pull request
+      name: PULL_BASE_SHA
+      type: string
+    - default: ""
+      description: git pull request number
+      name: PULL_NUMBER
+      type: string
+    - default: ""
+      description: git pull request ref in the form 'refs/pull/$PULL_NUMBER/head'
+      name: PULL_PULL_REF
+      type: string
+    - default: ""
+      description: git revision to checkout (branch, tag, sha, ref…)
+      name: PULL_PULL_SHA
+      type: string
+    - description: git pull reference strings of base and latest in the form 'master:$PULL_BASE_SHA,$PULL_NUMBER:$PULL_PULL_SHA:refs/pull/$PULL_NUMBER/head'
+      name: PULL_REFS
+      type: string
+    - description: git repository name
+      name: REPO_NAME
+      type: string
+    - description: git repository owner (user or organisation)
+      name: REPO_OWNER
+      type: string
+    - description: git url to clone
+      name: REPO_URL
+      type: string
+    tasks:
+    - name: from-build-pack
+      params:
+      - name: BUILD_ID
+        value: $(params.BUILD_ID)
+      - name: JOB_NAME
+        value: $(params.JOB_NAME)
+      - name: JOB_SPEC
+        value: $(params.JOB_SPEC)
+      - name: JOB_TYPE
+        value: $(params.JOB_TYPE)
+      - name: PULL_BASE_REF
+        value: $(params.PULL_BASE_REF)
+      - name: PULL_BASE_SHA
+        value: $(params.PULL_BASE_SHA)
+      - name: PULL_NUMBER
+        value: $(params.PULL_NUMBER)
+      - name: PULL_PULL_REF
+        value: $(params.PULL_PULL_REF)
+      - name: PULL_PULL_SHA
+        value: $(params.PULL_PULL_SHA)
+      - name: PULL_REFS
+        value: $(params.PULL_REFS)
+      - name: REPO_NAME
+        value: $(params.REPO_NAME)
+      - name: REPO_OWNER
+        value: $(params.REPO_OWNER)
+      - name: REPO_URL
+        value: $(params.REPO_URL)
+      taskSpec:
+        metadata: {}
+        params:
+        - description: the unique build number
+          name: BUILD_ID
+          type: string
+        - description: the name of the job which is the trigger context name
+          name: JOB_NAME
+          type: string
+        - description: the specification of the job
+          name: JOB_SPEC
+          type: string
+        - description: '''the kind of job: postsubmit or presubmit'''
+          name: JOB_TYPE
+          type: string
+        - description: the base git reference of the pull request
+          name: PULL_BASE_REF
+          type: string
+        - description: the git sha of the base of the pull request
+          name: PULL_BASE_SHA
+          type: string
+        - default: ""
+          description: git pull request number
+          name: PULL_NUMBER
+          type: string
+        - default: ""
+          description: git pull request ref in the form 'refs/pull/$PULL_NUMBER/head'
+          name: PULL_PULL_REF
+          type: string
+        - default: ""
+          description: git revision to checkout (branch, tag, sha, ref…)
+          name: PULL_PULL_SHA
+          type: string
+        - description: git pull reference strings of base and latest in the form 'master:$PULL_BASE_SHA,$PULL_NUMBER:$PULL_PULL_SHA:refs/pull/$PULL_NUMBER/head'
+          name: PULL_REFS
+          type: string
+        - description: git repository name
+          name: REPO_NAME
+          type: string
+        - description: git repository owner (user or organisation)
+          name: REPO_OWNER
+          type: string
+        - description: git url to clone
+          name: REPO_URL
+          type: string
+        stepTemplate:
+          env:
+          - name: BUILD_ID
+            value: $(params.BUILD_ID)
+          - name: JOB_NAME
+            value: $(params.JOB_NAME)
+          - name: JOB_SPEC
+            value: $(params.JOB_SPEC)
+          - name: JOB_TYPE
+            value: $(params.JOB_TYPE)
+          - name: PULL_BASE_REF
+            value: $(params.PULL_BASE_REF)
+          - name: PULL_BASE_SHA
+            value: $(params.PULL_BASE_SHA)
+          - name: PULL_NUMBER
+            value: $(params.PULL_NUMBER)
+          - name: PULL_PULL_REF
+            value: $(params.PULL_PULL_REF)
+          - name: PULL_PULL_SHA
+            value: $(params.PULL_PULL_SHA)
+          - name: PULL_REFS
+            value: $(params.PULL_REFS)
+          - name: REPO_NAME
+            value: $(params.REPO_NAME)
+          - name: REPO_OWNER
+            value: $(params.REPO_OWNER)
+          - name: REPO_URL
+            value: $(params.REPO_URL)
+          name: ""
+          resources:
+            requests:
+              cpu: 400m
+              memory: 512Mi
+          volumeMounts:
+          - mountPath: /tekton/home/npm
+            name: npmrc
+          workingDir: /workspace/source
+        steps:
+        - image: gcr.io/jenkinsxio/jx-boot:3.1.155
+          name: jx-variables
+          resources: {}
+          script: |
+            #!/usr/bin/env sh
+            jx gitops variables
+        - image: node:12-slim
+          name: build-npm-install
+          resources: {}
+          script: |
+            #!/bin/sh
+            npm install
+        - image: node:12-slim
+          name: build-npm-test
+          resources: {}
+          script: |
+            #!/bin/sh
+            CI=true DISPLAY=:99 npm test
+        - image: gcr.io/kaniko-project/executor:debug-v1.3.0
+          name: build-container-build
+          resources: {}
+          script: |
+            #!/busybox/sh
+            source .jx/variables.sh
+            cp /tekton/creds-secrets/tekton-container-registry-auth/.dockerconfigjson /kaniko/.docker/config.json
+            /kaniko/executor $KANIKO_FLAGS --context=/workspace/source --dockerfile=Dockerfile --destination=$DOCKER_REGISTRY/$DOCKER_REGISTRY_ORG/$APP_NAME:$VERSION
+        - image: gcr.io/jenkinsxio/jx-preview:0.0.143
+          name: promote-jx-preview
+          resources: {}
+          script: |
+            #!/usr/bin/env sh
+            source .jx/variables.sh
+            jx preview create
+        - image: ghcr.io/jenkins-x/jx-updatebot:0.0.82
+          name: promote-release
+          resources: {}
+          script: |
+            #!/usr/bin/env sh
+            source .jx/variables.sh
+            jx updatebot argo promote --target-git-url https://github.com/myorg/my-staging-repo
+        volumes:
+        - name: npmrc
+          secret:
+            optional: true
+            secretName: npmrc
+status: {}
diff --git a/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/source.yaml b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/source.yaml
new file mode 100644
index 00000000..4b7aaf99
--- /dev/null
+++ b/pkg/triggerconfig/inrepo/test_data/load_pipelinerun/uses-steps-override-script/source.yaml
@@ -0,0 +1,22 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+spec:
+  pipelineSpec:
+    tasks:
+    - name: from-build-pack
+      taskSpec:
+        stepTemplate:
+          image: uses:jenkins-x/jx3-pipeline-catalog/packs/javascript/.lighthouse/jenkins-x/pullrequest.yaml@1d39235ee9235d7d52d4025a8e59cb8bda04306a
+        steps:
+        - name: jx-variables
+        - image: uses:https://raw.githubusercontent.com/jenkins-x/jx3-pipeline-catalog/1d39235ee9235d7d52d4025a8e59cb8bda04306a/packs/javascript/.lighthouse/jenkins-x/pullrequest.yaml
+          name: build-npm-install
+        - name: build-npm-test
+        - name: build-container-build
+        - name: promote-jx-preview
+        - name: promote-release
+          script: |
+            #!/usr/bin/env sh
+            source .jx/variables.sh
+            jx updatebot argo promote --target-git-url https://github.com/myorg/my-staging-repo
+