Repository: mvisonneau/gitlab-ci-pipelines-exporter

Issue:
Add option to only pull open merge requests
Hey amazing tool you made here !

I was wondering if adding an option to only pull metrics for opened merge requests was a possibility as it would add tremendous value to the metrics collection on that side ?

Thanks and have a nice day

Discussion:

Solution patch:
diff --git a/pkg/config/project.go b/pkg/config/project.go
index c35ca4b..d70a6a0 100644
--- a/pkg/config/project.go
+++ b/pkg/config/project.go
@@ -87,12 +87,18 @@ type ProjectPullRefsMergeRequests struct {
 	// Monitor pipelines related to project merge requests.
 	Enabled bool `yaml:"enabled"`
 
+	// Filter for MRs to include (by title).
+	Regexp string `default:".*" yaml:"regexp"`
+
 	// Only keep most 'n' recently updated merge requests.
 	MostRecent uint `default:"0" yaml:"most_recent"`
 
 	// If the most recent pipeline for the merge request was last updated at
 	// time greater than this value the metrics won't be exported.
 	MaxAgeSeconds uint `default:"0" yaml:"max_age_seconds"`
+
+	// Prevent exporting metrics for MRs that are not opened.
+	ExcludeNonOpened bool `default:"false" yaml:"exclude_non_opened"`
 }
 
 // ProjectPullPipeline ..
diff --git a/pkg/gitlab/merge_requests.go b/pkg/gitlab/merge_requests.go
new file mode 100644
index 0000000..c71a647
--- /dev/null
+++ b/pkg/gitlab/merge_requests.go
@@ -0,0 +1,71 @@
+package gitlab
+
+import (
+	"context"
+	"regexp"
+
+	goGitlab "github.com/xanzy/go-gitlab"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
+)
+
+// GetProjectOpenMergeRequests ..
+func (c *Client) GetProjectOpenMergeRequests(ctx context.Context, p schemas.Project) (
+	mrs schemas.Refs,
+	err error,
+) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectOpenMergeRequests")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", p.Name))
+
+	mrs = make(schemas.Refs)
+
+	options := &goGitlab.ListProjectMergeRequestsOptions{
+		State: goGitlab.String("opened"),
+		ListOptions: goGitlab.ListOptions{
+			Page:    1,
+			PerPage: 100,
+		},
+	}
+
+	var re *regexp.Regexp
+
+	if re, err = regexp.Compile(p.Pull.Refs.MergeRequests.Regexp); err != nil {
+		return
+	}
+
+	for {
+		c.rateLimit(ctx)
+
+		var (
+			mrsList []*goGitlab.MergeRequest
+			resp    *goGitlab.Response
+		)
+
+		mrsList, resp, err = c.MergeRequests.ListProjectMergeRequests(p.Name, options, goGitlab.WithContext(ctx))
+		if err != nil {
+			return
+		}
+
+		c.requestsRemaining(resp)
+
+		for _, mr := range mrsList {
+			if re.MatchString(mr.Title) {
+				// Here the reference comes in with the ! prefix that we need to remove.
+				ref := mr.Reference[1:]
+				mr := schemas.NewRef(p, schemas.RefKindMergeRequest, ref)
+				mrs[mr.Key()] = mr
+			}
+		}
+
+		if resp.CurrentPage >= resp.TotalPages {
+			break
+		}
+
+		options.Page = resp.NextPage
+	}
+
+	return
+}
diff --git a/pkg/gitlab/pipelines.go b/pkg/gitlab/pipelines.go
index 4557c7a..781832a 100644
--- a/pkg/gitlab/pipelines.go
+++ b/pkg/gitlab/pipelines.go
@@ -167,15 +167,23 @@ func (c *Client) GetRefsFromPipelines(ctx context.Context, p schemas.Project, re
 	}
 
 	var (
-		mostRecent, maxAgeSeconds         uint
-		limitToMostRecent, excludeDeleted bool
-		existingRefs                      schemas.Refs
+		mostRecent, maxAgeSeconds                           uint
+		limitToMostRecent, excludeDeleted, excludeNonOpened bool
+		existingRefs                                        schemas.Refs
 	)
 
 	switch refKind {
 	case schemas.RefKindMergeRequest:
 		maxAgeSeconds = p.Pull.Refs.MergeRequests.MaxAgeSeconds
 		mostRecent = p.Pull.Refs.MergeRequests.MostRecent
+
+		if p.Pull.Refs.MergeRequests.ExcludeNonOpened {
+			excludeNonOpened = true
+
+			if existingRefs, err = c.GetProjectOpenMergeRequests(ctx, p); err != nil {
+				return
+			}
+		}
 	case schemas.RefKindBranch:
 		options.Scope = goGitlab.String("branches")
 		maxAgeSeconds = p.Pull.Refs.Branches.MaxAgeSeconds
@@ -252,7 +260,7 @@ func (c *Client) GetRefsFromPipelines(ctx context.Context, p schemas.Project, re
 				refName,
 			)
 
-			if excludeDeleted {
+			if excludeDeleted || excludeNonOpened {
 				if _, refExists := existingRefs[ref.Key()]; !refExists {
 					log.WithFields(log.Fields{
 						"project-name": ref.Project.Name,
diff --git a/pkg/schemas/metric.go b/pkg/schemas/metric.go
index bbf1975..9233a8f 100644
--- a/pkg/schemas/metric.go
+++ b/pkg/schemas/metric.go
@@ -155,7 +155,6 @@ func (m Metric) Key() MetricKey {
 			m.Labels["stage"],
 			m.Labels["tag_list"],
 			m.Labels["job_name"],
-			m.Labels["failure_reason"],
 		})
 
 	case MetricKindEnvironmentBehindCommitsCount, MetricKindEnvironmentBehindDurationSeconds, MetricKindEnvironmentDeploymentCount, MetricKindEnvironmentDeploymentDurationSeconds, MetricKindEnvironmentDeploymentJobID, MetricKindEnvironmentDeploymentStatus, MetricKindEnvironmentDeploymentTimestamp, MetricKindEnvironmentInformation:



Test patch:
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
index cb15964..1d6c5be 100644
--- a/pkg/config/config_test.go
+++ b/pkg/config/config_test.go
@@ -67,6 +67,8 @@ func TestNew(t *testing.T) {
 	c.ProjectDefaults.Pull.Refs.Tags.Regexp = `.*`
 	c.ProjectDefaults.Pull.Refs.Tags.ExcludeDeleted = true
 
+	c.ProjectDefaults.Pull.Refs.MergeRequests.Regexp = `.*`
+
 	c.ProjectDefaults.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true
 	c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true
 	c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
diff --git a/pkg/config/project_test.go b/pkg/config/project_test.go
index e4ef7bd..3dadad4 100644
--- a/pkg/config/project_test.go
+++ b/pkg/config/project_test.go
@@ -24,6 +24,8 @@ func TestNewProject(t *testing.T) {
 	p.Pull.Refs.Tags.Regexp = `.*`
 	p.Pull.Refs.Tags.ExcludeDeleted = true
 
+	p.Pull.Refs.MergeRequests.Regexp = `.*`
+
 	p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true
 	p.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true
 	p.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
diff --git a/pkg/config/wildcard_test.go b/pkg/config/wildcard_test.go
index 5b0afc7..289e841 100644
--- a/pkg/config/wildcard_test.go
+++ b/pkg/config/wildcard_test.go
@@ -22,6 +22,8 @@ func TestNewWildcard(t *testing.T) {
 	w.Pull.Refs.Tags.Regexp = `.*`
 	w.Pull.Refs.Tags.ExcludeDeleted = true
 
+	w.Pull.Refs.MergeRequests.Regexp = `.*`
+
 	w.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true
 	w.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true
 	w.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com`
diff --git a/pkg/gitlab/merge_requests_test.go b/pkg/gitlab/merge_requests_test.go
new file mode 100644
index 0000000..e2c8d95
--- /dev/null
+++ b/pkg/gitlab/merge_requests_test.go
@@ -0,0 +1,67 @@
+package gitlab
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
+)
+
+func TestGetProjectOpenMergeRequests(t *testing.T) {
+	ctx, mux, server, c := getMockedClient()
+	defer server.Close()
+
+	mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/merge_requests"),
+		func(w http.ResponseWriter, r *http.Request) {
+			assert.Equal(t, "GET", r.Method)
+			assert.Equal(t, []string{"100"}, r.URL.Query()["per_page"])
+			currentPage, err := strconv.Atoi(r.URL.Query()["page"][0])
+			assert.NoError(t, err)
+
+			nextPage := currentPage + 1
+
+			if currentPage == 2 {
+				nextPage = currentPage
+			}
+
+			w.Header().Add("X-Page", strconv.Itoa(currentPage))
+			w.Header().Add("X-Next-Page", strconv.Itoa(nextPage))
+
+			if currentPage == 1 {
+				fmt.Fprint(w, `[{"title":"Open MR", "reference":"!1"}]`)
+
+				return
+			}
+
+			fmt.Fprint(w, `[]`)
+		})
+
+	mux.HandleFunc(fmt.Sprintf("/api/v4/projects/0/merge_requests"),
+		func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotFound)
+		})
+
+	p := schemas.NewProject("foo")
+	expectedRef := schemas.NewRef(p, schemas.RefKindMergeRequest, "1")
+	refs, err := c.GetProjectOpenMergeRequests(ctx, p)
+	assert.NoError(t, err)
+	assert.Len(t, refs, 1)
+	assert.Equal(t, schemas.Refs{
+		expectedRef.Key(): expectedRef,
+	}, refs)
+
+	// Test invalid project name
+	p.Name = "invalid"
+	_, err = c.GetProjectOpenMergeRequests(ctx, p)
+	assert.Error(t, err)
+
+	// Test invalid regexp
+	p.Name = "foo"
+	p.Pull.Refs.MergeRequests.Regexp = `[`
+	_, err = c.GetProjectOpenMergeRequests(ctx, p)
+	assert.Error(t, err)
+}