Repository: goreleaser/goreleaser
Pull request #2573 (2021-10-12 18:50:01)

feat: auto-refresh checksums
This allows checksums to be "refreshed" at will.

The idea is, for instance, if your signing config replaces the original binary, you could have a config like this:

```yaml
signs:
# replaces the archives with signed ones
- cmd: something
  args: ["etc"]
  artifacts: archive
# refresh the checksums and sign with cosign
- cmd: cosign
  stdin: '{{ .Env.COSIGN_PWD }}'
  args: ["sign-blob", "-key=cosign.key", "-output=${signature}", "${artifact}"]
  artifacts: checksum
```

~~The pre-requisite is to sign the checksum as well to trigger the refresh. Maybe we can refresh it always in the end anyway?~~

So, this now actually refreshes the checksums once the sign pipe finishes... which means that the checksums should always be up to date.

closes  #1802


Patch:
diff --git a/internal/artifact/artifact.go b/internal/artifact/artifact.go
index 6655e504..521aa8ef 100644
--- a/internal/artifact/artifact.go
+++ b/internal/artifact/artifact.go
@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"crypto/sha512"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"hash"
 	"hash/crc32"
@@ -111,18 +112,35 @@ const (
 	ExtraFormat    = "Format"
 	ExtraWrappedIn = "WrappedIn"
 	ExtraBinaries  = "Binaries"
+	ExtraRefresh   = "Refresh"
 )
 
+// Extras represents the extra fields in an artifact.
+type Extras map[string]interface{}
+
+func (e Extras) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{}
+	for k, v := range e {
+		if k == ExtraRefresh {
+			// refresh is a func, so we can't serialize it.
+			// set v to a string representation of the function signature instead.
+			v = "func() error"
+		}
+		m[k] = v
+	}
+	return json.Marshal(m)
+}
+
 // Artifact represents an artifact and its relevant info.
 type Artifact struct {
-	Name   string                 `json:"name,omitempty"`
-	Path   string                 `json:"path,omitempty"`
-	Goos   string                 `json:"goos,omitempty"`
-	Goarch string                 `json:"goarch,omitempty"`
-	Goarm  string                 `json:"goarm,omitempty"`
-	Gomips string                 `json:"gomips,omitempty"`
-	Type   Type                   `json:"type,omitempty"`
-	Extra  map[string]interface{} `json:"extra,omitempty"`
+	Name   string `json:"name,omitempty"`
+	Path   string `json:"path,omitempty"`
+	Goos   string `json:"goos,omitempty"`
+	Goarch string `json:"goarch,omitempty"`
+	Goarm  string `json:"goarm,omitempty"`
+	Gomips string `json:"gomips,omitempty"`
+	Type   Type   `json:"type,omitempty"`
+	Extra  Extras `json:"extra,omitempty"`
 }
 
 // ExtraOr returns the Extra field with the given key or the or value specified
@@ -169,6 +187,25 @@ func (a Artifact) Checksum(algorithm string) (string, error) {
 	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
+var noRefresh = func() error { return nil }
+
+// Refresh executes a Refresh extra function on artifacts, if it exists.
+func (a Artifact) Refresh() error {
+	// for now lets only do it for checksums, as we know for a fact that
+	// they are the only ones that support this right now.
+	if a.Type != Checksum {
+		return nil
+	}
+	fn, ok := a.ExtraOr(ExtraRefresh, noRefresh).(func() error)
+	if !ok {
+		return nil
+	}
+	if err := fn(); err != nil {
+		return fmt.Errorf("failed to refresh %q: %w", a.Name, err)
+	}
+	return nil
+}
+
 // ID returns the artifact ID if it exists, empty otherwise.
 func (a Artifact) ID() string {
 	return a.ExtraOr(ExtraID, "").(string)
@@ -355,3 +392,16 @@ func (artifacts Artifacts) Paths() []string {
 	}
 	return result
 }
+
+// VisitFn is a function that can be executed against each artifact in a list.
+type VisitFn func(a *Artifact) error
+
+// Visit executes the given function for each artifact in the list.
+func (artifacts Artifacts) Visit(fn VisitFn) error {
+	for _, artifact := range artifacts.List() {
+		if err := fn(artifact); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/internal/pipe/blob/upload.go b/internal/pipe/blob/upload.go
index 40fbff6d..fea164aa 100644
--- a/internal/pipe/blob/upload.go
+++ b/internal/pipe/blob/upload.go
@@ -102,9 +102,7 @@ func doUpload(ctx *context.Context, conf config.Blob) error {
 			dataFile := artifact.Path
 			uploadFile := path.Join(folder, artifact.Name)
 
-			err := uploadData(ctx, conf, up, dataFile, uploadFile, bucketURL)
-
-			return err
+			return uploadData(ctx, conf, up, dataFile, uploadFile, bucketURL)
 		})
 	}
 
diff --git a/internal/pipe/checksums/checksums.go b/internal/pipe/checksums/checksums.go
index 373c9268..adeb11ba 100644
--- a/internal/pipe/checksums/checksums.go
+++ b/internal/pipe/checksums/checksums.go
@@ -3,6 +3,7 @@
 package checksums
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -17,6 +18,8 @@ import (
 	"github.com/goreleaser/goreleaser/pkg/context"
 )
 
+var errNoArtifacts = errors.New("there are no artifacts to sign")
+
 // Pipe for checksums.
 type Pipe struct{}
 
@@ -35,7 +38,33 @@ func (Pipe) Default(ctx *context.Context) error {
 }
 
 // Run the pipe.
-func (Pipe) Run(ctx *context.Context) (err error) {
+func (Pipe) Run(ctx *context.Context) error {
+	filename, err := tmpl.New(ctx).Apply(ctx.Config.Checksum.NameTemplate)
+	if err != nil {
+		return err
+	}
+	filepath := filepath.Join(ctx.Config.Dist, filename)
+	if err := refresh(ctx, filepath); err != nil {
+		if errors.Is(err, errNoArtifacts) {
+			return nil
+		}
+		return err
+	}
+	ctx.Artifacts.Add(&artifact.Artifact{
+		Type: artifact.Checksum,
+		Path: filepath,
+		Name: filename,
+		Extra: map[string]interface{}{
+			artifact.ExtraRefresh: func() error {
+				log.WithField("file", filename).Info("refreshing checksums")
+				return refresh(ctx, filepath)
+			},
+		},
+	})
+	return nil
+}
+
+func refresh(ctx *context.Context, filepath string) error {
 	filter := artifact.Or(
 		artifact.ByType(artifact.UploadableArchive),
 		artifact.ByType(artifact.UploadableBinary),
@@ -62,7 +91,7 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 	}
 
 	if len(artifactList) == 0 {
-		return nil
+		return errNoArtifacts
 	}
 
 	g := semerrgroup.New(ctx.Parallelism)
@@ -85,12 +114,8 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 		return err
 	}
 
-	filename, err := tmpl.New(ctx).Apply(ctx.Config.Checksum.NameTemplate)
-	if err != nil {
-		return err
-	}
 	file, err := os.OpenFile(
-		filepath.Join(ctx.Config.Dist, filename),
+		filepath,
 		os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
 		0o644,
 	)
@@ -99,12 +124,6 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 	}
 	defer file.Close()
 
-	ctx.Artifacts.Add(&artifact.Artifact{
-		Type: artifact.Checksum,
-		Path: file.Name(),
-		Name: filename,
-	})
-
 	// sort to ensure the signature is deterministic downstream
 	sort.Strings(sumLines)
 	_, err = file.WriteString(strings.Join(sumLines, ""))
diff --git a/internal/pipe/env/env.go b/internal/pipe/env/env.go
index 4fd17b07..e00315ad 100644
--- a/internal/pipe/env/env.go
+++ b/internal/pipe/env/env.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/apex/log"
 	"github.com/goreleaser/goreleaser/internal/tmpl"
@@ -19,7 +20,13 @@ var ErrMissingToken = errors.New("missing GITHUB_TOKEN, GITLAB_TOKEN and GITEA_T
 
 // ErrMultipleTokens indicates that multiple tokens are defined. ATM only one of them if allowed.
 // See https://github.com/goreleaser/goreleaser/pull/809
-var ErrMultipleTokens = errors.New("multiple tokens defined. Only one is allowed")
+type ErrMultipleTokens struct {
+	tokens []string
+}
+
+func (e ErrMultipleTokens) Error() string {
+	return fmt.Sprintf("multiple tokens found, but only one is allowed: %s\n\nLearn more at https://goreleaser.com/errors/multiple-tokens\n", strings.Join(e.tokens, ", "))
+}
 
 // Pipe for env.
 type Pipe struct{}
@@ -61,18 +68,18 @@ func (Pipe) Run(ctx *context.Context) error {
 	gitlabToken, gitlabTokenErr := loadEnv("GITLAB_TOKEN", ctx.Config.EnvFiles.GitLabToken)
 	giteaToken, giteaTokenErr := loadEnv("GITEA_TOKEN", ctx.Config.EnvFiles.GiteaToken)
 
-	numOfTokens := 0
+	var tokens []string
 	if githubToken != "" {
-		numOfTokens++
+		tokens = append(tokens, "GITHUB_TOKEN")
 	}
 	if gitlabToken != "" {
-		numOfTokens++
+		tokens = append(tokens, "GITLAB_TOKEN")
 	}
 	if giteaToken != "" {
-		numOfTokens++
+		tokens = append(tokens, "GITEA_TOKEN")
 	}
-	if numOfTokens > 1 {
-		return ErrMultipleTokens
+	if len(tokens) > 1 {
+		return ErrMultipleTokens{tokens}
 	}
 
 	noTokens := githubToken == "" && gitlabToken == "" && giteaToken == ""
diff --git a/internal/pipe/sign/sign.go b/internal/pipe/sign/sign.go
index 1691e9d1..5bbb4fe1 100644
--- a/internal/pipe/sign/sign.go
+++ b/internal/pipe/sign/sign.go
@@ -95,11 +95,22 @@ func (Pipe) Run(ctx *context.Context) error {
 			return sign(ctx, cfg, ctx.Artifacts.Filter(artifact.And(filters...)).List())
 		})
 	}
-	return g.Wait()
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	return ctx.Artifacts.
+		Filter(artifact.ByType(artifact.Checksum)).
+		Visit(func(a *artifact.Artifact) error {
+			return a.Refresh()
+		})
 }
 
 func sign(ctx *context.Context, cfg config.Sign, artifacts []*artifact.Artifact) error {
 	for _, a := range artifacts {
+		if err := a.Refresh(); err != nil {
+			return err
+		}
 		artifacts, err := signone(ctx, cfg, a)
 		if err != nil {
 			return err
diff --git a/www/docs/errors/multiple-tokens.md b/www/docs/errors/multiple-tokens.md
new file mode 100644
index 00000000..49dda230
--- /dev/null
+++ b/www/docs/errors/multiple-tokens.md
@@ -0,0 +1,15 @@
+# Multiple tokens found, but only one is allowed
+
+GoReleaser infers if you are using GitHub, GitLab or Gitea by which tokens are provided.
+If you have multiple tokens set, you'll get this error.
+
+Here's an example:
+
+```sh
+   ⨯ release failed after 0.02s error=gmultiple tokens found, but only one is allowed: GITHUB_TOKEN, GITLAB_TOKEN
+
+Learn more at https://goreleaser.com/errors/multiple-tokens
+```
+
+In this case, you either unset `GITHUB_TOKEN` or `GITLAB_TOKEN`.
+You can read more about it in the [SCM docs](/scm/github/).
diff --git a/www/mkdocs.yml b/www/mkdocs.yml
index 935ea9d9..9a57b474 100644
--- a/www/mkdocs.yml
+++ b/www/mkdocs.yml
@@ -131,6 +131,7 @@ nav:
     - goreleaser jsonschema: cmd/goreleaser_jsonschema.md
 - Common errors:
   - errors/dirty.md
+  - errors/multiple-tokens.md
   - errors/no-main.md
 - deprecations.md
 - Cookbooks:


Test patch:
diff --git a/internal/artifact/artifact_test.go b/internal/artifact/artifact_test.go
index 17afade3..827b5e78 100644
--- a/internal/artifact/artifact_test.go
+++ b/internal/artifact/artifact_test.go
@@ -1,11 +1,13 @@
 package artifact
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/goreleaser/goreleaser/internal/golden"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 )
@@ -390,3 +392,123 @@ func TestPaths(t *testing.T) {
 	}
 	require.ElementsMatch(t, paths, artifacts.Paths())
 }
+
+func TestRefresher(t *testing.T) {
+	t.Run("ok", func(t *testing.T) {
+		artifacts := New()
+		path := filepath.Join(t.TempDir(), "f")
+		artifacts.Add(&Artifact{
+			Name: "f",
+			Path: path,
+			Type: Checksum,
+			Extra: map[string]interface{}{
+				"Refresh": func() error {
+					return os.WriteFile(path, []byte("hello"), 0o765)
+				},
+			},
+		})
+		artifacts.Add(&Artifact{
+			Name: "invalid",
+			Type: Checksum,
+			Extra: map[string]interface{}{
+				"Refresh": func() {
+					t.Fatalf("should not have been called")
+				},
+			},
+		})
+		artifacts.Add(&Artifact{
+			Name: "no refresh",
+			Type: Checksum,
+		})
+
+		for _, item := range artifacts.List() {
+			require.NoError(t, item.Refresh())
+		}
+
+		bts, err := os.ReadFile(path)
+		require.NoError(t, err)
+		require.Equal(t, "hello", string(bts))
+	})
+
+	t.Run("nok", func(t *testing.T) {
+		artifacts := New()
+		artifacts.Add(&Artifact{
+			Name: "fail",
+			Type: Checksum,
+			Extra: map[string]interface{}{
+				"ID": "nok",
+				"Refresh": func() error {
+					return fmt.Errorf("fake err")
+				},
+			},
+		})
+
+		for _, item := range artifacts.List() {
+			require.EqualError(t, item.Refresh(), `failed to refresh "fail": fake err`)
+		}
+	})
+
+	t.Run("not a checksum", func(t *testing.T) {
+		artifacts := New()
+		artifacts.Add(&Artifact{
+			Name: "will be ignored",
+			Type: Binary,
+			Extra: map[string]interface{}{
+				"ID": "ignored",
+				"Refresh": func() error {
+					return fmt.Errorf("err that should not happen")
+				},
+			},
+		})
+
+		for _, item := range artifacts.List() {
+			require.NoError(t, item.Refresh())
+		}
+	})
+}
+
+func TestVisit(t *testing.T) {
+	artifacts := New()
+	artifacts.Add(&Artifact{
+		Name: "foo",
+		Type: Checksum,
+	})
+	artifacts.Add(&Artifact{
+		Name: "foo",
+		Type: Binary,
+	})
+
+	t.Run("ok", func(t *testing.T) {
+		require.NoError(t, artifacts.Visit(func(a *Artifact) error {
+			require.Equal(t, "foo", a.Name)
+			return nil
+		}))
+	})
+
+	t.Run("nok", func(t *testing.T) {
+		require.EqualError(t, artifacts.Visit(func(a *Artifact) error {
+			return fmt.Errorf("fake err")
+		}), `fake err`)
+	})
+}
+
+func TestMarshalJSON(t *testing.T) {
+	artifacts := New()
+	artifacts.Add(&Artifact{
+		Name: "foo",
+		Type: Binary,
+		Extra: map[string]interface{}{
+			ExtraID: "adsad",
+		},
+	})
+	artifacts.Add(&Artifact{
+		Name: "foo",
+		Type: Checksum,
+		Extra: map[string]interface{}{
+			ExtraRefresh: func() error { return nil },
+		},
+	})
+	bts, err := json.Marshal(artifacts.List())
+	require.NoError(t, err)
+	golden.RequireEqualJSON(t, bts)
+}
diff --git a/internal/artifact/testdata/TestMarshalJSON.json.golden b/internal/artifact/testdata/TestMarshalJSON.json.golden
new file mode 100644
index 00000000..a0091ac8
--- /dev/null
+++ b/internal/artifact/testdata/TestMarshalJSON.json.golden
@@ -0,0 +1,1 @@
+[{"name":"foo","type":"Binary","extra":{"ID":"adsad"}},{"name":"foo","type":"Checksum","extra":{"Refresh":"func() error"}}]
\ No newline at end of file
diff --git a/internal/pipe/checksums/checksums_test.go b/internal/pipe/checksums/checksums_test.go
index e324cf34..6568672e 100644
--- a/internal/pipe/checksums/checksums_test.go
+++ b/internal/pipe/checksums/checksums_test.go
@@ -3,6 +3,7 @@ package checksums
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/goreleaser/goreleaser/internal/artifact"
@@ -20,27 +21,28 @@ func TestPipe(t *testing.T) {
 	const archive = binary + ".tar.gz"
 	const linuxPackage = binary + ".rpm"
 	const checksums = binary + "_bar_checksums.txt"
+	const sum = "61d034473102d7dac305902770471fd50f4c5b26f6831a56dd90b5184b3c30fc  "
 
 	tests := map[string]struct {
 		ids  []string
-		want []string
+		want string
 	}{
 		"default": {
-			want: []string{
-				binary,
-				archive,
-				linuxPackage,
-			},
+			want: strings.Join([]string{
+				sum + binary,
+				sum + linuxPackage,
+				sum + archive,
+			}, "\n") + "\n",
 		},
 		"select ids": {
 			ids: []string{
 				"id-1",
 				"id-2",
 			},
-			want: []string{
-				binary,
-				archive,
-			},
+			want: strings.Join([]string{
+				sum + binary,
+				sum + archive,
+			}, "\n") + "\n",
 		},
 	}
 	for name, tt := range tests {
@@ -89,17 +91,50 @@ func TestPipe(t *testing.T) {
 			var artifacts []string
 			for _, a := range ctx.Artifacts.List() {
 				artifacts = append(artifacts, a.Name)
+				require.NoError(t, a.Refresh(), "refresh should not fail and yield same results as nothing changed")
 			}
 			require.Contains(t, artifacts, checksums, binary)
 			bts, err := os.ReadFile(filepath.Join(folder, checksums))
 			require.NoError(t, err)
-			for _, want := range tt.want {
-				require.Contains(t, string(bts), "61d034473102d7dac305902770471fd50f4c5b26f6831a56dd90b5184b3c30fc  "+want)
-			}
+			require.Contains(t, tt.want, string(bts))
 		})
 	}
 }
 
+func TestRefreshModifying(t *testing.T) {
+	const binary = "binary"
+	folder := t.TempDir()
+	file := filepath.Join(folder, binary)
+	require.NoError(t, os.WriteFile(file, []byte("some string"), 0o644))
+	ctx := context.New(
+		config.Project{
+			Dist:        folder,
+			ProjectName: binary,
+			Checksum: config.Checksum{
+				NameTemplate: "{{ .ProjectName }}_{{ .Env.FOO }}_checksums.txt",
+				Algorithm:    "sha256",
+			},
+		},
+	)
+	ctx.Git.CurrentTag = "1.2.3"
+	ctx.Env = map[string]string{"FOO": "bar"}
+	ctx.Artifacts.Add(&artifact.Artifact{
+		Name: binary,
+		Path: file,
+		Type: artifact.UploadableBinary,
+	})
+	require.NoError(t, Pipe{}.Run(ctx))
+	checks := ctx.Artifacts.Filter(artifact.ByType(artifact.Checksum)).List()
+	require.Len(t, checks, 1)
+	previous, err := os.ReadFile(checks[0].Path)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(file, []byte("some other string"), 0o644))
+	require.NoError(t, checks[0].Refresh())
+	current, err := os.ReadFile(checks[0].Path)
+	require.NoError(t, err)
+	require.NotEqual(t, string(previous), string(current))
+}
+
 func TestPipeFileNotExist(t *testing.T) {
 	folder := t.TempDir()
 	ctx := context.New(
@@ -188,7 +223,9 @@ func TestPipeCouldNotOpenChecksumsTxt(t *testing.T) {
 }
 
 func TestPipeWhenNoArtifacts(t *testing.T) {
-	ctx := &context.Context{}
+	ctx := &context.Context{
+		Artifacts: artifact.New(),
+	}
 	require.NoError(t, Pipe{}.Run(ctx))
 	require.Len(t, ctx.Artifacts.List(), 0)
 }
diff --git a/internal/pipe/env/env_test.go b/internal/pipe/env/env_test.go
index bd198dfc..c4b98172 100644
--- a/internal/pipe/env/env_test.go
+++ b/internal/pipe/env/env_test.go
@@ -120,7 +120,7 @@ func TestMultipleEnvTokens(t *testing.T) {
 		Config: config.Project{},
 	}
 	require.Error(t, Pipe{}.Run(ctx))
-	require.EqualError(t, Pipe{}.Run(ctx), ErrMultipleTokens.Error())
+	require.EqualError(t, Pipe{}.Run(ctx), "multiple tokens found, but only one is allowed: GITHUB_TOKEN, GITLAB_TOKEN, GITEA_TOKEN\n\nLearn more at https://goreleaser.com/errors/multiple-tokens\n")
 	// so the tests do not depend on each other
 	require.NoError(t, os.Unsetenv("GITHUB_TOKEN"))
 	require.NoError(t, os.Unsetenv("GITLAB_TOKEN"))
diff --git a/internal/pipe/sign/sign_test.go b/internal/pipe/sign/sign_test.go
index 91e19570..c34bb8d2 100644
--- a/internal/pipe/sign/sign_test.go
+++ b/internal/pipe/sign/sign_test.go
@@ -612,6 +612,12 @@ func testSign(tb testing.TB, ctx *context.Context, certificateNames, signaturePa
 		Name: "checksum2",
 		Path: filepath.Join(tmpdir, "checksum2"),
 		Type: artifact.Checksum,
+		Extra: map[string]interface{}{
+			"Refresh": func() error {
+				file := filepath.Join(tmpdir, "checksum2")
+				return os.WriteFile(file, []byte("foo"), 0o644)
+			},
+		},
 	})
 	ctx.Artifacts.Add(&artifact.Artifact{
 		Name: "artifact4_1.0.0_linux_amd64",