Repository: jenkins-x/lighthouse
Pull request #1349 (2021-07-07 14:43:44)

fix: add webhook poller


Patch:
diff --git a/.lighthouse/jenkins-x/pullrequest.yaml b/.lighthouse/jenkins-x/pullrequest.yaml
index 445fbc6d..340409b4 100755
--- a/.lighthouse/jenkins-x/pullrequest.yaml
+++ b/.lighthouse/jenkins-x/pullrequest.yaml
@@ -35,6 +35,13 @@ spec:
             source .jx/variables.sh
             cp /tekton/creds-secrets/tekton-container-registry-auth/.dockerconfigjson /kaniko/.docker/config.json
             /kaniko/executor $KANIKO_FLAGS --context=/workspace/source --dockerfile=docker/webhooks/Dockerfile --destination=ghcr.io/jenkins-x/lighthouse-webhooks:$VERSION --build-arg=VERSION=$VERSION
+        - name: build-container-build:poller
+          resources: {}
+          script: |
+            #!/busybox/sh
+            source .jx/variables.sh
+            cp /tekton/creds-secrets/tekton-container-registry-auth/.dockerconfigjson /kaniko/.docker/config.json
+            /kaniko/executor $KANIKO_FLAGS --context=/workspace/source --dockerfile=docker/poller/Dockerfile --destination=ghcr.io/jenkins-x/lighthouse-poller:$VERSION --build-arg=VERSION=$VERSION
         - name: build-container-build:keeper
           resources: {}
           script: |
diff --git a/.lighthouse/jenkins-x/release.yaml b/.lighthouse/jenkins-x/release.yaml
index 8fb0c8c7..c7e7b20c 100755
--- a/.lighthouse/jenkins-x/release.yaml
+++ b/.lighthouse/jenkins-x/release.yaml
@@ -32,6 +32,13 @@ spec:
             source .jx/variables.sh
             cp /tekton/creds-secrets/tekton-container-registry-auth/.dockerconfigjson /kaniko/.docker/config.json
             /kaniko/executor $KANIKO_FLAGS --context=/workspace/source --dockerfile=docker/webhooks/Dockerfile --destination=ghcr.io/jenkins-x/lighthouse-webhooks:$VERSION --destination=ghcr.io/jenkins-x/lighthouse-webhooks:latest --build-arg=VERSION=$VERSION
+        - name: build-and-push-image:poller
+          resources: {}
+          script: |
+            #!/busybox/sh
+            source .jx/variables.sh
+            cp /tekton/creds-secrets/tekton-container-registry-auth/.dockerconfigjson /kaniko/.docker/config.json
+            /kaniko/executor $KANIKO_FLAGS --context=/workspace/source --dockerfile=docker/poller/Dockerfile --destination=ghcr.io/jenkins-x/lighthouse-poller:$VERSION --destination=ghcr.io/jenkins-x/lighthouse-poller:latest --build-arg=VERSION=$VERSION
         - name: build-and-push-image:keeper
           resources: {}
           script: |
diff --git a/Makefile b/Makefile
index 96f7aa79..2bf839fb 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ rwildcard=$(wildcard $1$2) $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2))
 PROJECT := github.com/jenkins-x/lighthouse
 
 WEBHOOKS_EXECUTABLE := webhooks
+POLLER_EXECUTABLE := poller
 KEEPER_EXECUTABLE := keeper
 FOGHORN_EXECUTABLE := foghorn
 GC_JOBS_EXECUTABLE := gc-jobs
@@ -11,6 +12,7 @@ TEKTON_CONTROLLER_EXECUTABLE := lighthouse-tekton-controller
 JENKINS_CONTROLLER_EXECUTABLE := jenkins-controller
 
 WEBHOOKS_MAIN_SRC_FILE=cmd/webhooks/main.go
+POLLER_MAIN_SRC_FILE=cmd/poller/main.go
 KEEPER_MAIN_SRC_FILE=cmd/keeper/main.go
 FOGHORN_MAIN_SRC_FILE=cmd/foghorn/main.go
 GC_JOBS_MAIN_SRC_FILE=cmd/gc/main.go
@@ -30,12 +32,16 @@ GO_DEPENDENCIES := $(call rwildcard,pkg/,*.go) $(call rwildcard,cmd/,*.go)
 all: build test check docs ## Default rule, builds all binaries, runs tests and format checks
 
 .PHONY: build
-build: build-webhooks build-keeper build-foghorn build-tekton-controller build-gc-jobs build-jenkins-controller ## Builds all Lighthouse binaries native to your machine
+build: build-webhooks build-poller build-keeper build-foghorn build-tekton-controller build-gc-jobs build-jenkins-controller ## Builds all Lighthouse binaries native to your machine
 
 .PHONY: build-webhooks
 build-webhooks: ## Build the webhooks controller binary for the native OS
 	$(GO) build -i -ldflags "$(GO_LDFLAGS)" -o bin/$(WEBHOOKS_EXECUTABLE) $(WEBHOOKS_MAIN_SRC_FILE)
 
+.PHONY: build-poller
+build-poller: ## Build the poller controller binary for the native OS
+	$(GO) build -i -ldflags "$(GO_LDFLAGS)" -o bin/$(POLLER_EXECUTABLE) $(POLLER_MAIN_SRC_FILE)
+
 .PHONY: build-keeper
 build-keeper: ## Build the keeper controller binary for the native OS
 	$(GO) build -i -ldflags "$(GO_LDFLAGS)" -o bin/$(KEEPER_EXECUTABLE) $(KEEPER_MAIN_SRC_FILE)
@@ -63,12 +69,16 @@ release: linux
 linux: build-linux
 
 .PHONY: build-linux
-build-linux: build-webhooks-linux build-foghorn-linux build-gc-jobs-linux build-keeper-linux build-tekton-controller-linux build-jenkins-controller-linux ## Build all binaries for Linux
+build-linux: build-webhooks-linux build-poller-linux build-foghorn-linux build-gc-jobs-linux build-keeper-linux build-tekton-controller-linux build-jenkins-controller-linux ## Build all binaries for Linux
 
 .PHONY: build-webhooks-linux ## Build the webhook controller binary for Linux
 build-webhooks-linux:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO) build -ldflags "$(GO_LDFLAGS)" -o bin/$(WEBHOOKS_EXECUTABLE) $(WEBHOOKS_MAIN_SRC_FILE)
 
+.PHONY: build-poller-linux ## Build the webhook controller binary for Linux
+build-poller-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO) build -ldflags "$(GO_LDFLAGS)" -o bin/$(POLLER_EXECUTABLE) $(POLLER_MAIN_SRC_FILE)
+
 .PHONY: build-keeper-linux
 build-keeper-linux: ## Build the keeper controller binary for Linux
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO) build -ldflags "$(GO_LDFLAGS)" -o bin/$(KEEPER_EXECUTABLE) $(KEEPER_MAIN_SRC_FILE)
diff --git a/charts/lighthouse/templates/_helpers.tpl b/charts/lighthouse/templates/_helpers.tpl
index 1755371f..ae42bf7c 100644
--- a/charts/lighthouse/templates/_helpers.tpl
+++ b/charts/lighthouse/templates/_helpers.tpl
@@ -21,6 +21,11 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Chart.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{- define "poller.name" -}}
+{{- $name := default "poller" .Values.poller.nameOverride -}}
+{{- printf "%s-%s" .Chart.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{- define "foghorn.name" -}}
 {{- $name := default "foghorn" .Values.foghorn.nameOverride -}}
 {{- printf "%s-%s" .Chart.Name $name | trunc 63 | trimSuffix "-" -}}
diff --git a/charts/lighthouse/templates/poller-deployment.yaml b/charts/lighthouse/templates/poller-deployment.yaml
new file mode 100644
index 00000000..b3e05bbe
--- /dev/null
+++ b/charts/lighthouse/templates/poller-deployment.yaml
@@ -0,0 +1,123 @@
+{{- if .Values.poller.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "poller.name" . }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    app: {{ template "poller.name" . }}
+spec:
+  replicas: {{ .Values.poller.replicaCount }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: {{ template "poller.name" . }}
+  template:
+    metadata:
+{{- if or .Values.poller.datadog.enabled .Values.poller.podAnnotations }}
+      annotations:
+{{- if .Values.poller.datadog.enabled }}
+        ad.datadoghq.com/poller.logs: '[{"source":"lighthouse","service":"poller"}]'
+{{- end }}
+{{- if .Values.poller.podAnnotations }}
+{{ toYaml .Values.poller.podAnnotations | indent 8 }}
+{{- end }}
+{{- end }}
+      labels:
+        app: {{ template "poller.name" . }}
+    spec:
+      serviceAccountName: {{ template "poller.name" . }}
+      terminationGracePeriodSeconds: {{ .Values.poller.terminationGracePeriodSeconds }}
+      containers:
+      - name: {{ template "poller.name" . }}
+        image: {{ tpl .Values.poller.image.repository . }}:{{ tpl .Values.poller.image.tag . }}
+        imagePullPolicy: {{ tpl .Values.poller.image.pullPolicy . }}
+        args:
+          - "--namespace={{ .Release.Namespace }}"
+        ports:
+          - name: http
+            containerPort: {{ .Values.poller.service.internalPort }}
+            protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: {{ .Values.poller.probe.path }}
+            port: http
+          initialDelaySeconds: {{ .Values.poller.livenessProbe.initialDelaySeconds }}
+          periodSeconds: {{ .Values.poller.livenessProbe.periodSeconds }}
+          successThreshold: {{ .Values.poller.livenessProbe.successThreshold }}
+          timeoutSeconds: {{ .Values.poller.livenessProbe.timeoutSeconds }}
+        readinessProbe:
+          httpGet:
+            path: {{ .Values.poller.probe.path }}
+            port: http
+          periodSeconds: {{ .Values.poller.readinessProbe.periodSeconds }}
+          successThreshold: {{ .Values.poller.readinessProbe.successThreshold }}
+          timeoutSeconds: {{ .Values.poller.readinessProbe.timeoutSeconds }}
+        env:
+        - name: "GIT_KIND"
+          value: "{{ .Values.git.kind }}"
+        - name: "GIT_SERVER"
+          value: "{{ .Values.git.server }}"
+{{- if .Values.githubApp.enabled }}
+        - name: "GITHUB_APP_SECRET_DIR"
+          value: "/secrets/githubapp/tokens"
+{{- else }}
+        - name: "GIT_USER"
+          value: {{ .Values.user }}
+        - name: "GIT_TOKEN"
+          valueFrom:
+            secretKeyRef:
+              name: lighthouse-oauth-token
+              key: oauth
+{{- end }}
+        - name: "JX_LOG_FORMAT"
+          value: "{{ .Values.logFormat }}"
+        - name: "LOGRUS_FORMAT"
+          value: "{{ .Values.logFormat }}"
+        - name: LOGRUS_SERVICE
+          value: "{{ .Values.logService | default .Chart.Name }}"
+        - name: LOGRUS_SERVICE_VERSION
+          value: "{{ .Chart.Version }}"
+        - name: LOGRUS_STACK_SKIP
+          value: "{{ .Values.logStackSkip }}"
+{{- if hasKey .Values.poller "env" }}
+{{- range $pkey, $pval := .Values.poller.env }}
+        - name: {{ $pkey }}
+          value: {{ quote  $pval }}
+{{- end }}
+{{- end }}
+        envFrom:
+        - secretRef:
+            name: jx-boot-job-env-vars
+            optional: true
+        resources:
+{{ toYaml .Values.poller.resources | indent 10 }}
+{{- if .Values.githubApp.enabled }}
+        volumeMounts:
+        - name: githubapp-tokens
+          mountPath: /secrets/githubapp/tokens
+          readOnly: true
+{{- end }}
+{{- if .Values.githubApp.enabled }}
+      volumes:
+      - name: githubapp-tokens
+        secret:
+          secretName: tide-githubapp-tokens
+{{- end }}
+{{- with .Values.poller.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.poller.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.poller.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- end }}
diff --git a/charts/lighthouse/templates/poller-rb.yaml b/charts/lighthouse/templates/poller-rb.yaml
new file mode 100644
index 00000000..56acb0d1
--- /dev/null
+++ b/charts/lighthouse/templates/poller-rb.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.poller.enabled }}
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "poller.name" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "poller.name" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ template "poller.name" . }}
+{{- end }}
diff --git a/charts/lighthouse/templates/poller-role.yaml b/charts/lighthouse/templates/poller-role.yaml
new file mode 100644
index 00000000..53284312
--- /dev/null
+++ b/charts/lighthouse/templates/poller-role.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.poller.enabled }}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "poller.name" . }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+{{- end }}
diff --git a/charts/lighthouse/templates/poller-sa.yaml b/charts/lighthouse/templates/poller-sa.yaml
new file mode 100644
index 00000000..e3e94173
--- /dev/null
+++ b/charts/lighthouse/templates/poller-sa.yaml
@@ -0,0 +1,6 @@
+{{- if .Values.poller.enabled }}
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: {{ template "poller.name" . }}
+{{- end }}
diff --git a/charts/lighthouse/values.yaml b/charts/lighthouse/values.yaml
index e5e8bbd2..2fd7c371 100644
--- a/charts/lighthouse/values.yaml
+++ b/charts/lighthouse/values.yaml
@@ -396,6 +396,78 @@ keeper:
   # keeper.tolerations -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) applied to the keeper pods
   tolerations: []
 
+poller:
+  # poller.enabled -- Whether to enable or disable the poller component
+  enabled: false
+
+  # poller.replicaCount -- Number of replicas
+  replicaCount: 1
+
+  # poller.terminationGracePeriodSeconds -- Termination grace period for poller pods
+  terminationGracePeriodSeconds: 30
+
+  image:
+    # poller.image.repository -- Template for computing the poller controller docker image repository
+    repository: "{{ .Values.image.parentRepository }}/lighthouse-poller"
+
+    # poller.image.tag -- Template for computing the poller controller docker image tag
+    tag: "{{ .Values.image.tag }}"
+
+    # poller.image.pullPolicy -- Template for computing the poller controller docker image pull policy
+    pullPolicy: "{{ .Values.image.pullPolicy }}"
+
+  # poller.podAnnotations -- Annotations applied to the poller pods
+  podAnnotations: {}
+
+  # poller.env -- Lets you define poller specific environment variables
+  env:
+    # poller.env.POLL_PERIOD the time period before polling
+    POLL_PERIOD: 20s
+
+    # poller.env.POLL_HOOK_ENDPOINT the hook service endpoint to post webhooks to
+    POLL_HOOK_ENDPOINT: http://hook/hook/poll
+
+  resources:
+    # poller.resources.limits -- Resource limits applied to the poller pods
+    limits:
+      cpu: 400m
+      memory: 512Mi
+
+    # poller.resources.requests -- Resource requests applied to the poller pods
+    requests:
+      cpu: 100m
+      memory: 128Mi
+
+  # poller.probe -- Liveness and readiness probes settings
+  probe:
+    path: /
+
+  # poller.livenessProbe -- Liveness probe configuration
+  livenessProbe:
+    initialDelaySeconds: 120
+    periodSeconds: 10
+    successThreshold: 1
+    timeoutSeconds: 1
+
+  # poller.readinessProbe -- Readiness probe configuration
+  readinessProbe:
+    periodSeconds: 10
+    successThreshold: 1
+    timeoutSeconds: 1
+
+  datadog:
+    # poller.datadog.enabled -- Enables datadog
+    enabled: "true"
+
+  # poller.nodeSelector -- [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) applied to the poller pods
+  nodeSelector: {}
+
+  # poller.affinity -- [Affinity rules](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) applied to the poller pods
+  affinity: {}
+
+  # poller.tolerations -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) applied to the poller pods
+  tolerations: []
+
 engines:
   # engines.jx -- Enables the jx engine
   jx: true
diff --git a/cmd/poller/main.go b/cmd/poller/main.go
new file mode 100644
index 00000000..453652b1
--- /dev/null
+++ b/cmd/poller/main.go
@@ -0,0 +1,256 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"github.com/jenkins-x/go-scm/scm"
+	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
+	gitv2 "github.com/jenkins-x/lighthouse/pkg/git/v2"
+	"github.com/jenkins-x/lighthouse/pkg/poller"
+	"github.com/pkg/errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/jenkins-x/lighthouse/pkg/config"
+	configutil "github.com/jenkins-x/lighthouse/pkg/config/util"
+	"github.com/jenkins-x/lighthouse/pkg/interrupts"
+	"github.com/jenkins-x/lighthouse/pkg/jobutil"
+	"github.com/jenkins-x/lighthouse/pkg/logrusutil"
+	"github.com/jenkins-x/lighthouse/pkg/util"
+	"github.com/jenkins-x/lighthouse/pkg/watcher"
+	"github.com/sirupsen/logrus"
+)
+
+type options struct {
+	configPath    string
+	jobConfigPath string
+	botName       string
+	gitServerURL  string
+	gitKind       string
+	namespace     string
+	repoNames     string
+	hookEndpoint  string
+	runOnce       bool
+	dryRun        bool
+	pollPeriod    time.Duration
+}
+
+func (o *options) Validate() error {
+	return nil
+}
+
+func gatherOptions(fs *flag.FlagSet, args ...string) options {
+	var o options
+	fs.StringVar(&o.configPath, "config-path", "", "Path to config.yaml.")
+	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
+	fs.StringVar(&o.botName, "bot-name", "", "The bot name")
+	fs.StringVar(&o.gitServerURL, "git-url", "", "The git provider URL")
+	fs.StringVar(&o.gitKind, "git-kind", "", "The git provider kind (e.g. github, gitlab, bitbucketserver")
+	fs.BoolVar(&o.runOnce, "run-once", false, "If true, run only once then quit.")
+	fs.BoolVar(&o.dryRun, "dry-run", false, "Disable POSTing to the webhook service and just log the webhooks instead.")
+
+	fs.StringVar(&o.namespace, "namespace", "jx", "The namespace to listen in")
+	fs.StringVar(&o.repoNames, "repo", "", "The git repository names to poll. If not specified all the repositories are polled")
+	fs.StringVar(&o.hookEndpoint, "hook", os.Getenv("POLL_HOOK_ENDPOINT"), "The hook endpoint to post to")
+
+	defaultPollPeriod := 20 * time.Second
+	text := os.Getenv("POLL_PERIOD")
+	if text != "" {
+		d, err := time.ParseDuration(text)
+		if err != nil {
+			logrus.WithError(err).WithField("PollPeriod", text).Warn("invalid POLL_PERIOD value")
+		} else {
+			defaultPollPeriod = d
+		}
+	}
+	fs.DurationVar(&o.pollPeriod, "period", defaultPollPeriod, "The time period between polls")
+
+	err := fs.Parse(args)
+	if err != nil {
+		logrus.WithError(err).Fatal("Invalid options")
+	}
+	o.configPath = configutil.PathOrDefault(o.configPath)
+	return o
+}
+
+func main() {
+	logrusutil.ComponentInit("poller")
+
+	defer interrupts.WaitForGracefulShutdown()
+
+	jobutil.ServePProf()
+
+	o := gatherOptions(flag.NewFlagSet(os.Args[0], flag.ExitOnError), os.Args[1:]...)
+	if err := o.Validate(); err != nil {
+		logrus.WithError(err).Fatal("Invalid options")
+	}
+
+	configAgent := &config.Agent{}
+	cfgMapWatcher, err := watcher.SetupConfigMapWatchers(o.namespace, configAgent, nil)
+	if err != nil {
+		logrus.WithError(err).Fatal("error starting config map watcher")
+	}
+	defer cfgMapWatcher.Stop()
+
+	botName := o.botName
+	if botName == "" {
+		botName = util.GetBotName(configAgent.Config)
+	}
+	if util.GetGitHubAppSecretDir() != "" {
+		botName, err = util.GetGitHubAppAPIUser()
+		if err != nil {
+			logrus.WithError(err).Fatal("unable to read API user for GitHub App integration")
+		}
+	}
+	if botName == "" {
+		logrus.Fatal("no $GIT_USER defined")
+	}
+	if o.hookEndpoint == "" {
+		logrus.Fatal("no hook endpoint defined")
+	}
+	serverURL := o.gitServerURL
+	if serverURL == "" {
+		serverURL = util.GetGitServer(configAgent.Config)
+	}
+	gitKind := o.gitKind
+	if gitKind == "" {
+		gitKind = util.GitKind(configAgent.Config)
+	}
+	gitToken, err := util.GetSCMToken(gitKind)
+	if err != nil {
+		logrus.WithError(err).Fatal("Error creating Poller controller.")
+	}
+
+	gitCloneUser := o.botName
+
+	u, err := url.Parse(serverURL)
+	if err != nil {
+		logrus.WithError(err).Fatalf("failed to parse git server %s", serverURL)
+	}
+
+	configureOpts := func(opts *gitv2.ClientFactoryOpts) {
+		opts.Token = func() []byte {
+			return []byte(gitToken)
+		}
+		opts.GitUser = func() (name, email string, err error) {
+			name = gitCloneUser
+			return
+		}
+		opts.Username = func() (login string, err error) {
+			login = gitCloneUser
+			return
+		}
+		opts.Host = u.Host
+		opts.Scheme = u.Scheme
+	}
+	gitFactory, err := gitv2.NewClientFactory(configureOpts)
+	if err != nil {
+		logrus.WithError(err).Fatalf("failed to create git client factory for server %s", o.gitServerURL)
+	}
+	fb := filebrowser.NewFileBrowserFromGitClient(gitFactory)
+
+	var repoNames []string
+	if o.repoNames != "" {
+		repoNames = strings.Split(o.repoNames, ",")
+	}
+	if len(repoNames) == 0 {
+		cfg := configAgent.Config
+		repoNames = findAllRepoNames(cfg())
+	}
+	if len(repoNames) == 0 {
+		logrus.Fatal("no repositories found")
+	}
+
+	gitHubAppOwner := ""
+	_, scmClient, _, _, err := util.GetSCMClient(gitHubAppOwner, configAgent.Config)
+	if err != nil {
+		logrus.WithError(err).Fatal("failed to create scm client")
+	}
+
+	c, err := poller.NewPollingController(repoNames, serverURL, scmClient, fb, o.notifier)
+	if err != nil {
+		logrus.WithError(err).Fatal("Error creating Poller controller.")
+	}
+
+	c.Logger().WithFields(map[string]interface{}{
+		"PollPeriod":  o.pollPeriod,
+		"HookEndpint": o.hookEndpoint,
+	}).Info("starting")
+
+	start := time.Now()
+	c.Sync()
+	if o.runOnce {
+		return
+	}
+
+	// run the controller, but only after one sync period expires after our first run
+	time.Sleep(time.Until(start.Add(o.pollPeriod)))
+	interrupts.Tick(func() {
+		c.Sync()
+	}, func() time.Duration {
+		return o.pollPeriod
+	})
+}
+
+func findAllRepoNames(c *config.Config) []string {
+	m := map[string]bool{}
+
+	for fullName := range c.Presubmits {
+		m[fullName] = true
+	}
+	for fullName := range c.Postsubmits {
+		m[fullName] = true
+	}
+	for fullName := range c.InRepoConfig.Enabled {
+		if !strings.Contains(fullName, "*") && strings.Contains(fullName, "/") {
+			m[fullName] = true
+		}
+	}
+	var repoNames []string
+	for fullName := range m {
+		repoNames = append(repoNames, fullName)
+	}
+	return repoNames
+}
+
+func (o *options) notifier(hook *scm.WebhookWrapper) error {
+	if o.dryRun {
+		logrus.WithField("Hook", hook).Info("notify")
+		return nil
+	}
+
+	data, err := json.Marshal(hook)
+	if err != nil {
+		return errors.Wrapf(err, "failed to marshal hook %#v", hook)
+	}
+
+	req, err := http.NewRequest("POST", o.hookEndpoint, bytes.NewBuffer(data))
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		if err != nil {
+			return errors.Wrapf(err, "failed to invoke endpoint %s", o.hookEndpoint)
+		}
+	}
+	defer resp.Body.Close()
+
+	body, _ := ioutil.ReadAll(resp.Body)
+	l := logrus.WithFields(map[string]interface{}{
+		"Status":   resp.Status,
+		"Endpoint": o.hookEndpoint,
+		"Body":     string(body),
+	})
+	if resp.StatusCode >= 300 {
+		l.Warnf("failed to notify")
+		return errors.Errorf("status when invoking endpoint")
+	}
+	l.Infof("notified")
+	return nil
+}
diff --git a/cmd/webhooks/main.go b/cmd/webhooks/main.go
index a468d9fc..efbb83c9 100644
--- a/cmd/webhooks/main.go
+++ b/cmd/webhooks/main.go
@@ -22,6 +22,7 @@ const (
 type options struct {
 	bindAddress string
 	path        string
+	pollPath    string
 	port        int
 	jsonLog     bool
 
@@ -42,7 +43,9 @@ func gatherOptions(fs *flag.FlagSet, args ...string) options {
 	fs.StringVar(&o.bindAddress, "bind", "",
 		"The interface address to bind to (by default, will listen on all interfaces/addresses).")
 	fs.StringVar(&o.path, "path", "/hook",
-		"The path to listen on for requests to trigger a pipeline run.")
+		"The path to listen on for webhook to trigger a pipeline run.")
+	fs.StringVar(&o.pollPath, "pollPath", "/hook/poll",
+		"The path to listen on for polling requests to trigger a pipeline run.")
 	fs.StringVar(&o.pluginFilename, "plugin-file", "", "Path to the plugins.yaml file. If not specified it is loaded from the 'plugins' ConfigMap")
 	fs.StringVar(&o.configFilename, "config-file", "", "Path to the config.yaml file. If not specified it is loaded from the 'config' ConfigMap")
 	fs.StringVar(&o.botName, "bot-name", "", "The name of the bot user to run as. Defaults to $GIT_USER if not specified.")
@@ -84,6 +87,7 @@ func main() {
 
 	mux.Handle("/", http.HandlerFunc(controller.DefaultHandler))
 	mux.Handle(o.path, http.HandlerFunc(controller.HandleWebhookRequests))
+	mux.Handle(o.pollPath, http.HandlerFunc(controller.HandlePollingRequests))
 
 	// lets serve metrics
 	metricsHandler := http.HandlerFunc(controller.Metrics)
diff --git a/docker/poller/Dockerfile b/docker/poller/Dockerfile
new file mode 100644
index 00000000..75c740b2
--- /dev/null
+++ b/docker/poller/Dockerfile
@@ -0,0 +1,10 @@
+FROM alpine:3.12
+
+RUN apk add --update --no-cache ca-certificates git \
+    && adduser -D -u 1000 jx
+
+ENV JX_HOME /home/jx
+USER 1000
+
+COPY ./bin/poller /home/jx/
+ENTRYPOINT ["/home/jx/poller"]
diff --git a/pkg/poller/poller.go b/pkg/poller/poller.go
new file mode 100644
index 00000000..ece93cb4
--- /dev/null
+++ b/pkg/poller/poller.go
@@ -0,0 +1,307 @@
+package poller
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"github.com/jenkins-x/go-scm/scm"
+	"github.com/jenkins-x/lighthouse/pkg/filebrowser"
+	"github.com/jenkins-x/lighthouse/pkg/git/v2"
+	"github.com/jenkins-x/lighthouse/pkg/poller/pollstate"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	censor = func(content []byte) []byte { return content }
+)
+
+type pollingController struct {
+	repositoryNames []string
+	gitServer       string
+	scmClient       *scm.Client
+	fb              filebrowser.Interface
+	pollstate       pollstate.Interface
+	logger          *logrus.Entry
+	notifier        func(webhook *scm.WebhookWrapper) error
+}
+
+func NewPollingController(repositoryNames []string, gitServer string, scmClient *scm.Client, fb filebrowser.Interface, notifier func(webhook *scm.WebhookWrapper) error) (*pollingController, error) {
+	logger := logrus.NewEntry(logrus.StandardLogger())
+	if gitServer == "" {
+		gitServer = "https://github.com"
+	}
+	return &pollingController{
+		repositoryNames: repositoryNames,
+		gitServer:       gitServer,
+		logger:          logger,
+		scmClient:       scmClient,
+		fb:              fb,
+		notifier:        notifier,
+		pollstate:       pollstate.NewMemoryPollState(),
+	}, nil
+}
+
+func (c *pollingController) Logger() *logrus.Entry {
+	return c.logger
+}
+
+func (c *pollingController) Sync() {
+	c.PollReleases()
+	c.PollPullRequests()
+}
+
+func (c *pollingController) PollReleases() {
+	ctx := context.TODO()
+
+	for _, fullName := range c.repositoryNames {
+		l := c.logger.WithField("Repo", fullName)
+
+		l.Info("polling for new commit on main branch")
+
+		// lets git clone and see if the latest git commit sha is new...
+		owner, repo := scm.Split(fullName)
+		ref := ""
+		fc := filebrowser.NewFetchCache()
+		err := c.fb.WithDir(owner, repo, ref, fc, func(dir string) error {
+			executor, err := git.NewCensoringExecutor(dir, censor, l)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create git executor")
+			}
+
+			out, err := executor.Run("rev-parse", "HEAD")
+			if err != nil {
+				return errors.Wrapf(err, "failed to get latest git commit sha")
+			}
+			sha := strings.TrimSpace(string(out))
+			if sha == "" {
+				return errors.Errorf("could not find latest git commit sha")
+			}
+			out, err = executor.Run("rev-parse", "--abbrev-ref", "HEAD")
+			if err != nil {
+				return errors.Wrapf(err, "failed to get current git branch")
+			}
+			branch := strings.TrimSpace(string(out))
+
+			l = l.WithField("SHA", sha).WithField("Branch", branch)
+
+			newValue, err := c.pollstate.IsNew(fullName, "release", sha)
+			if err != nil {
+				return errors.Wrapf(err, "failed to check if sha %s is new", sha)
+			}
+			if !newValue {
+				return nil
+			}
+
+			// lets check we have not triggered this before
+			hasStatus, err := c.hasStatusForSHA(ctx, l, fullName, sha)
+			if err != nil {
+				return errors.Wrapf(err, "failed to check for status ")
+			}
+			if hasStatus {
+				return nil
+			}
+
+			l.Infof("triggering release webhook")
+
+			before := ""
+			pushHook, err := c.createPushHook(fullName, owner, repo, before, sha, branch, branch)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create PushHook")
+			}
+
+			wh := &scm.WebhookWrapper{
+				PushHook: pushHook,
+			}
+			err = c.notifier(wh)
+			if err != nil {
+				return errors.Wrapf(err, "failed to notify PushHook")
+			}
+			l.Infof("notified PushHook")
+			return nil
+		})
+		if err != nil {
+			l.WithError(err).Warn("failed to poll release")
+		}
+	}
+}
+
+func (c *pollingController) PollPullRequests() {
+	ctx := context.TODO()
+
+	for _, fullName := range c.repositoryNames {
+		l := c.logger.WithField("Repo", fullName)
+
+		l.Info("polling for new commit on main branch")
+
+		opts := scm.PullRequestListOptions{
+			Open: true,
+
+			// TODO use last update poll?
+		}
+		prs, _, err := c.scmClient.PullRequests.List(ctx, fullName, opts)
+		if err != nil {
+			l.WithError(err).Error("failed to list open pull requests")
+			continue
+		}
+		if len(prs) == 0 {
+			l.Info("no open Pull Requests")
+			continue
+		}
+
+		for _, pr := range prs {
+			sha := pr.Sha
+			if sha == "" {
+				l.Infof("no SHA for PullRequest")
+				continue
+			}
+			prName := "PR-" + strconv.Itoa(pr.Number)
+
+			l2 := l.WithFields(map[string]interface{}{
+				"SHA":         sha,
+				"PullRequest": pr.Number,
+			})
+			err = c.pollPullRequest(ctx, l2, fullName, pr, prName, sha)
+			if err != nil {
+				l2.WithError(err).Error("failed to check for PullRequestHook")
+				continue
+			}
+			err = c.pollPullRequestPushHook(ctx, l2, fullName, pr, prName, sha)
+			if err != nil {
+				l2.WithError(err).Error("failed to check for PullRequestHook")
+				continue
+			}
+		}
+	}
+}
+
+func (c *pollingController) pollPullRequest(ctx context.Context, l *logrus.Entry, fullName string, pr *scm.PullRequest, prName, sha string) error {
+	newValue, err := c.pollstate.IsNew(fullName, prName, "created")
+	if err != nil {
+		return errors.Wrapf(err, "failed to check if sha %s is new", sha)
+	}
+	if !newValue {
+		return nil
+	}
+
+	hasStatus, err := c.hasStatusForSHA(ctx, l, fullName, sha)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check for status ")
+	}
+	if hasStatus {
+		return nil
+	}
+
+	l.Infof("triggering pull request webhook")
+
+	pullRequestHook, err := c.createPullRequestHook(fullName, pr)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create PullRequestHook")
+	}
+
+	wh := &scm.WebhookWrapper{
+		PullRequestHook: pullRequestHook,
+	}
+	err = c.notifier(wh)
+	if err != nil {
+		return errors.Wrapf(err, "failed to notify PullRequestHook")
+	}
+	l.Infof("notified PullRequestHook")
+	return nil
+}
+
+func (c *pollingController) pollPullRequestPushHook(ctx context.Context, l *logrus.Entry, fullName string, pr *scm.PullRequest, prName, sha string) error {
+	newValue, err := c.pollstate.IsNew(fullName, prName+"-push", sha)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check if sha %s is new", sha)
+	}
+	if !newValue {
+		return nil
+	}
+
+	hasStatus, err := c.hasStatusForSHA(ctx, l, fullName, sha)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check for status ")
+	}
+	if hasStatus {
+		return nil
+	}
+
+	l.Infof("triggering pull request push webhook")
+
+	owner, repo := scm.Split(fullName)
+	branch := pr.Repository().Branch
+	if branch == "" {
+		branch = pr.Head.Ref
+	}
+	l = l.WithField("Branch", branch)
+	before := ""
+	refBranch := pr.Source
+	pushHook, err := c.createPushHook(fullName, owner, repo, before, sha, branch, refBranch)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create PushHook")
+	}
+
+	wh := &scm.WebhookWrapper{
+		PushHook: pushHook,
+	}
+	err = c.notifier(wh)
+	if err != nil {
+		return errors.Wrapf(err, "failed to notify PR PushHook")
+	}
+	l.Infof("notified PR PushHook")
+	return nil
+}
+
+func (c *pollingController) hasStatusForSHA(ctx context.Context, l *logrus.Entry, fullName string, sha string) (bool, error) {
+	opts := scm.ListOptions{
+		Page: 1,
+	}
+	statuses, _, err := c.scmClient.Repositories.ListStatus(ctx, fullName, sha, opts)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to list status")
+	}
+	for _, s := range statuses {
+		if !strings.HasPrefix(s.Label, "Lighthouse") {
+			l.WithField("Statuses", statuses).Info("the SHA has CI statuses so not triggering")
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (c *pollingController) createPushHook(fullName, owner, repo, before, after, branch, refBranch string) (*scm.PushHook, error) {
+	return &scm.PushHook{
+		Ref: "refs/heads/" + refBranch,
+		Repo: scm.Repository{
+			Namespace: owner,
+			Name:      repo,
+			FullName:  fullName,
+			Branch:    branch,
+			Clone:     c.gitServer + "/" + fullName + ".git",
+			Link:      c.gitServer + "/" + fullName,
+		},
+		Before:       before,
+		After:        after,
+		Commits:      nil,
+		Commit:       scm.Commit{},
+		Sender:       scm.User{},
+		GUID:         after,
+		Installation: nil,
+	}, nil
+}
+
+func (c *pollingController) createPullRequestHook(fullName string, pr *scm.PullRequest) (*scm.PullRequestHook, error) {
+	repo := pr.Repository()
+	return &scm.PullRequestHook{
+		Action:       scm.ActionOpen,
+		Repo:         repo,
+		Label:        scm.Label{},
+		PullRequest:  *pr,
+		Sender:       scm.User{},
+		Changes:      scm.PullRequestHookChanges{},
+		GUID:         "",
+		Installation: nil,
+	}, nil
+}
diff --git a/pkg/poller/pollstate/interface.go b/pkg/poller/pollstate/interface.go
new file mode 100644
index 00000000..289e9c6e
--- /dev/null
+++ b/pkg/poller/pollstate/interface.go
@@ -0,0 +1,5 @@
+package pollstate
+
+type Interface interface {
+	IsNew(repository, operation, values string) (bool, error)
+}
diff --git a/pkg/poller/pollstate/memory.go b/pkg/poller/pollstate/memory.go
new file mode 100644
index 00000000..ac3baa91
--- /dev/null
+++ b/pkg/poller/pollstate/memory.go
@@ -0,0 +1,27 @@
+package pollstate
+
+import "sync"
+
+type memoryPollstate struct {
+	cache map[string]string
+	lock  sync.RWMutex
+}
+
+// NewMemoryPollState creates a new poll state
+func NewMemoryPollState() Interface {
+	return &memoryPollstate{
+		cache: map[string]string{},
+	}
+}
+
+func (p *memoryPollstate) IsNew(repository, operation, newValue string) (bool, error) {
+	key := repository + ":" + operation
+
+	p.lock.Lock()
+
+	old := p.cache[key]
+	p.cache[key] = newValue
+
+	p.lock.Unlock()
+	return old != newValue, nil
+}
diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go
index 77986082..6cc40dd6 100644
--- a/pkg/webhook/webhook.go
+++ b/pkg/webhook/webhook.go
@@ -2,10 +2,12 @@ package webhook
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 
 	"github.com/jenkins-x/lighthouse/pkg/externalplugincfg"
@@ -42,6 +44,7 @@ type WebhooksController struct {
 	gitClient               git.Client
 	launcher                launcher.PipelineLauncher
 	disabledExternalPlugins []string
+	logWebHooks             bool
 }
 
 // NewWebhooksController creates and configures the controller
@@ -52,6 +55,10 @@ func NewWebhooksController(path, namespace, botName, pluginFilename, configFilen
 		pluginFilename: pluginFilename,
 		configFilename: configFilename,
 		botName:        botName,
+		logWebHooks:    os.Getenv("LIGHTHOUSE_LOG_WEBHOOKS") == "true",
+	}
+	if o.logWebHooks {
+		logrus.Info("enabling webhook logging")
 	}
 	var err error
 	o.server, err = o.createHookServer()
@@ -123,8 +130,31 @@ func (o *WebhooksController) isReady() bool {
 	return true
 }
 
-// HandleWebhookRequests handles incoming events
+// HandleWebhookRequests handles incoming webhook events
 func (o *WebhooksController) HandleWebhookRequests(w http.ResponseWriter, r *http.Request) {
+	o.handleWebhookOrPollRequest(w, r, "Webhook", func(scmClient *scm.Client, r *http.Request) (scm.Webhook, error) {
+		return scmClient.Webhooks.Parse(r, o.secretFn)
+	})
+}
+
+// HandlePollingRequests handles incoming polling events
+func (o *WebhooksController) HandlePollingRequests(w http.ResponseWriter, r *http.Request) {
+	o.handleWebhookOrPollRequest(w, r, "Pollhook", func(scmClient *scm.Client, r *http.Request) (scm.Webhook, error) {
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to read poll payload")
+		}
+		wh := &scm.WebhookWrapper{}
+		err = json.Unmarshal(data, wh)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to unmarshal WebhookWrapper payload")
+		}
+		return wh.ToWebhook()
+	})
+}
+
+// handleWebhookOrPollRequest handles incoming events
+func (o *WebhooksController) handleWebhookOrPollRequest(w http.ResponseWriter, r *http.Request, operation string, parseWebhook func(scmClient *scm.Client, r *http.Request) (scm.Webhook, error)) {
 	if r.Method != http.MethodPost {
 		// liveness probe etc
 		logrus.WithField("method", r.Method).Debug("invalid http method so returning 200")
@@ -156,7 +186,7 @@ func (o *WebhooksController) HandleWebhookRequests(w http.ResponseWriter, r *htt
 		return
 	}
 
-	webhook, err := scmClient.Webhooks.Parse(r, o.secretFn)
+	webhook, err := parseWebhook(scmClient, r)
 	if err != nil {
 		logrus.Warnf("failed to parse webhook: %s", err.Error())
 
@@ -245,7 +275,7 @@ func (o *WebhooksController) HandleWebhookRequests(w http.ResponseWriter, r *htt
 			return
 		}
 	}
-	entry := logrus.WithField("Webhook", webhook.Kind())
+	entry := logrus.WithField(operation, webhook.Kind())
 	if o.disabledExternalPlugins == nil {
 		o.disabledExternalPlugins, err = externalplugincfg.LoadDisabledPlugins(entry, kubeClient, o.namespace)
 		if err != nil {
@@ -291,6 +321,11 @@ func (o *WebhooksController) ProcessWebHook(l *logrus.Entry, webhook scm.Webhook
 	}
 
 	l = l.WithFields(fields)
+
+	if o.logWebHooks {
+		l.WithField("WebHook", webhook).Info("webhook")
+	}
+
 	_, ok := webhook.(*scm.PingHook)
 	if ok {
 		l.Info("received ping")


Test patch:
diff --git a/pkg/poller/poller_test.go b/pkg/poller/poller_test.go
new file mode 100644
index 00000000..c1a4010f
--- /dev/null
+++ b/pkg/poller/poller_test.go
@@ -0,0 +1,102 @@
+package poller_test
+
+import (
+	scmfake "github.com/jenkins-x/go-scm/scm/driver/fake"
+	"testing"
+
+	fbfake "github.com/jenkins-x/lighthouse/pkg/filebrowser/fake"
+	"github.com/jenkins-x/lighthouse/pkg/poller"
+
+	"github.com/jenkins-x/go-scm/scm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	repoNames = []string{"myorg/myrepo"}
+
+	gitServer = "https://github.com"
+)
+
+func TestPollerReleases(t *testing.T) {
+	var hooks []*scm.PushHook
+
+	fakeNotifier := func(wrapper *scm.WebhookWrapper) error {
+		hook := wrapper.PushHook
+		assert.NotNil(t, hook, "no PushHook for webhook %#v", wrapper)
+		if hook != nil {
+			hooks = append(hooks, hook)
+		}
+		return nil
+	}
+	scmClient, _ := scmfake.NewDefault()
+	fb := fbfake.NewFakeFileBrowser("test_data", true)
+
+	p, err := poller.NewPollingController(repoNames, gitServer, scmClient, fb, fakeNotifier)
+	require.NoError(t, err, "failed to create PollingController")
+
+	p.PollReleases()
+
+	require.Len(t, hooks, 1, "should have 1 PushHook")
+
+	hook := hooks[0]
+	assert.NotNil(t, hook, "no PushHook")
+	t.Logf("created PushHook %#v", hook)
+}
+
+func TestPollerPullRequests(t *testing.T) {
+	var prHooks []*scm.PullRequestHook
+	var pushHooks []*scm.PushHook
+
+	fakeNotifier := func(wrapper *scm.WebhookWrapper) error {
+		if wrapper.PullRequestHook != nil {
+			prHooks = append(prHooks, wrapper.PullRequestHook)
+		} else if wrapper.PushHook != nil {
+			pushHooks = append(pushHooks, wrapper.PushHook)
+		} else {
+			assert.Fail(t, "unknown webhook %v", wrapper)
+		}
+		return nil
+	}
+	scmClient, fakeData := scmfake.NewDefault()
+	fb := fbfake.NewFakeFileBrowser("test_data", true)
+
+	prNumber := 123
+	fullName := repoNames[0]
+	owner, repo := scm.Split(fullName)
+	sha := "mysha1234"
+	fakeData.PullRequests[prNumber] = &scm.PullRequest{
+		Number: prNumber,
+		Base: scm.PullRequestBranch{
+			Ref: "master",
+			Repo: scm.Repository{
+				Namespace: owner,
+				Name:      repo,
+				FullName:  fullName,
+			},
+		},
+		Head: scm.PullRequestBranch{
+			Sha: sha,
+		},
+		Title:  "fix: some stuff",
+		Body:   "the PR comment",
+		Closed: false,
+		State:  "open",
+		Sha:    sha,
+	}
+	p, err := poller.NewPollingController(repoNames, gitServer, scmClient, fb, fakeNotifier)
+	require.NoError(t, err, "failed to create PollingController")
+
+	p.PollPullRequests()
+
+	require.Len(t, prHooks, 1, "should have 1 PullRequestHook")
+	require.Len(t, pushHooks, 1, "should have 1 PushHook")
+
+	hook := prHooks[0]
+	assert.NotNil(t, hook, "no PullRequestHook")
+	t.Logf("created PullRequestHook %#v", hook)
+
+	pushHook := pushHooks[0]
+	assert.NotNil(t, pushHook, "no PushHook")
+	t.Logf("created PushHook %#v", pushHook)
+}
diff --git a/pkg/poller/test_data/myorg/myrepo/README.md b/pkg/poller/test_data/myorg/myrepo/README.md
new file mode 100644
index 00000000..8ef16a54
--- /dev/null
+++ b/pkg/poller/test_data/myorg/myrepo/README.md
@@ -0,0 +1,1 @@
+dummy content in git
\ No newline at end of file
diff --git a/pkg/triggerconfig/inrepo/calculate_test.go b/pkg/triggerconfig/inrepo/calculate_test.go
index 0d7edb22..d4db402d 100644
--- a/pkg/triggerconfig/inrepo/calculate_test.go
+++ b/pkg/triggerconfig/inrepo/calculate_test.go
@@ -1,7 +1,6 @@
 package inrepo_test
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/sirupsen/logrus"
@@ -41,7 +40,7 @@ func TestCalculate(t *testing.T) {
 	}
 	sharedPluginConfig := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser("test_data", true))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
@@ -121,7 +120,7 @@ func TestTriggersInBranchMergeToMaster(t *testing.T) {
 	}
 	sharedPluginConfig := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser("test_data", true))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
@@ -168,7 +167,7 @@ func TestIssue1306(t *testing.T) {
 	}
 	sharedPluginConfig := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser("test_data", true))
 	require.NoError(t, err, "failed to create filebrowsers")
 	fc := filebrowser.NewFetchCache()
 
diff --git a/pkg/triggerconfig/inrepo/load_integration_test.go b/pkg/triggerconfig/inrepo/load_integration_test.go
index 4b121b97..0a074eea 100644
--- a/pkg/triggerconfig/inrepo/load_integration_test.go
+++ b/pkg/triggerconfig/inrepo/load_integration_test.go
@@ -2,7 +2,6 @@ package inrepo
 
 import (
 	"os"
-	"path/filepath"
 	"testing"
 
 	fbfake "github.com/jenkins-x/lighthouse/pkg/filebrowser/fake"
@@ -27,7 +26,7 @@ func TestMergeConfigIntegration(t *testing.T) {
 	cfg := &config.Config{}
 	pluginCfg := &plugins.Configuration{}
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser("test_data", true))
 	require.NoError(t, err, "failed to create filebrowsers")
 
 	fc := filebrowser.NewFetchCache()
diff --git a/pkg/triggerconfig/inrepo/load_pipelinerun_test.go b/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
index f557683f..b0396619 100644
--- a/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
+++ b/pkg/triggerconfig/inrepo/load_pipelinerun_test.go
@@ -31,7 +31,7 @@ func TestLoadPipelineRunTest(t *testing.T) {
 	fs, err := ioutil.ReadDir(sourceDir)
 	require.NoError(t, err, "failed to read source Dir %s", sourceDir)
 
-	fileBrowser := fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true)
+	fileBrowser := fbfake.NewFakeFileBrowser("test_data", true)
 
 	// lets use a custom version stream sha
 	os.Setenv("LIGHTHOUSE_VERSIONSTREAM_JENKINS_X_JX3_PIPELINE_CATALOG", "myversionstreamref")
diff --git a/pkg/triggerconfig/inrepo/load_triggers_test.go b/pkg/triggerconfig/inrepo/load_triggers_test.go
index e251dcdd..9930eff2 100644
--- a/pkg/triggerconfig/inrepo/load_triggers_test.go
+++ b/pkg/triggerconfig/inrepo/load_triggers_test.go
@@ -21,7 +21,7 @@ func TestMergeConfig(t *testing.T) {
 	repo := "loadtest"
 	ref := "master"
 
-	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+	fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser("test_data", true))
 	require.NoError(t, err, "failed to create filebrowsers")
 
 	cfg := &config.Config{}
@@ -44,7 +44,7 @@ func TestInvalidConfigs(t *testing.T) {
 		owner := "myorg"
 		ref := "master"
 
-		fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser(filepath.Join("test_data"), true))
+		fileBrowsers, err := filebrowser.NewFileBrowsers(filebrowser.GitHubURL, fbfake.NewFakeFileBrowser("test_data", true))
 		require.NoError(t, err, "failed to create filebrowsers")
 
 		fc := filebrowser.NewFetchCache()
diff --git a/pkg/triggerconfig/merge/merge_test.go b/pkg/triggerconfig/merge/merge_test.go
index a2df0ce8..521a250d 100644
--- a/pkg/triggerconfig/merge/merge_test.go
+++ b/pkg/triggerconfig/merge/merge_test.go
@@ -77,7 +77,7 @@ func TestMergeTriggerConfig(t *testing.T) {
 }
 
 func TestMergeTriggerConfigFiles(t *testing.T) {
-	sourceData := filepath.Join("test_data")
+	sourceData := "test_data"
 	fileNames, err := ioutil.ReadDir(sourceData)
 	assert.NoError(t, err)
 
diff --git a/pkg/webhook/create_agent_integration_test.go b/pkg/webhook/create_agent_integration_test.go
index 7e4fd8e4..fe07fcdd 100644
--- a/pkg/webhook/create_agent_integration_test.go
+++ b/pkg/webhook/create_agent_integration_test.go
@@ -1,6 +1,9 @@
 package webhook
 
 import (
+	"os"
+	"testing"
+
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jenkins-x/go-scm/scm"
 	"github.com/jenkins-x/lighthouse/pkg/clients"
@@ -14,8 +17,6 @@ import (
 	"github.com/jenkins-x/lighthouse/pkg/watcher"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
-	"os"
-	"testing"
 )
 
 func TestCreateAgentIntegration(t *testing.T) {