Repository: goreleaser/goreleaser
Pull request #3343 (2022-08-25 04:50:19)

fix: source archive add subfolders
`--add-file` adds only the `--prefix` joined with the basename of the added file, so, adding a folder like `vendor` would break havoc.

this basically writes our own source archives (which are more compressed) and allows to add an entire folder easily, as well as other mappings as `archives` already supports. 

refs #3102 #2911


codecov[bot]: # [Codecov](https://codecov.io/gh/goreleaser/goreleaser/pull/3343?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser) Report
> Merging [#3343](https://codecov.io/gh/goreleaser/goreleaser/pull/3343?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser) (3ce21a9) into [main](https://codecov.io/gh/goreleaser/goreleaser/commit/12c425614c16d894f0e4999533540474ba167891?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser) (12c4256) will **decrease** coverage by `0.16%`.
> The diff coverage is `52.94%`.

> :exclamation: Current head 3ce21a9 differs from pull request most recent head fa87dec. Consider uploading reports for the commit fa87dec to get more accurate results

```diff
@@            Coverage Diff             @@
##             main    #3343      +/-   ##
==========================================
- Coverage   83.93%   83.76%   -0.17%     
==========================================
  Files         115      115              
  Lines        9210     9211       +1     
==========================================
- Hits         7730     7716      -14     
- Misses       1204     1215      +11     
- Partials      276      280       +4     
```


| [Impacted Files](https://codecov.io/gh/goreleaser/goreleaser/pull/3343?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser) | Coverage Δ | |
|---|---|---|
| [pkg/config/config.go](https://codecov.io/gh/goreleaser/goreleaser/pull/3343/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser#diff-cGtnL2NvbmZpZy9jb25maWcuZ28=) | `95.21% <ø> (ø)` | |
| [internal/pipe/sourcearchive/source.go](https://codecov.io/gh/goreleaser/goreleaser/pull/3343/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser#diff-aW50ZXJuYWwvcGlwZS9zb3VyY2VhcmNoaXZlL3NvdXJjZS5nbw==) | `67.56% <52.94%> (-20.11%)` | :arrow_down: |

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=goreleaser)


Patch:
diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go
index 339d718e..102713bb 100644
--- a/internal/pipe/sourcearchive/source.go
+++ b/internal/pipe/sourcearchive/source.go
@@ -2,13 +2,17 @@
 package sourcearchive
 
 import (
+	"fmt"
+	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/caarlos0/log"
 	"github.com/goreleaser/goreleaser/internal/archivefiles"
 	"github.com/goreleaser/goreleaser/internal/artifact"
 	"github.com/goreleaser/goreleaser/internal/git"
 	"github.com/goreleaser/goreleaser/internal/tmpl"
+	"github.com/goreleaser/goreleaser/pkg/archive"
 	"github.com/goreleaser/goreleaser/pkg/config"
 	"github.com/goreleaser/goreleaser/pkg/context"
 )
@@ -33,31 +37,54 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 	filename := name + "." + ctx.Config.Source.Format
 	path := filepath.Join(ctx.Config.Dist, filename)
 	log.WithField("file", filename).Info("creating source archive")
-	args := []string{
-		"archive",
-		"-o", path,
-		"--format", ctx.Config.Source.Format,
+
+	out, err := git.Run(ctx, "ls-files")
+	if err != nil {
+		return fmt.Errorf("could not list source files: %w", err)
 	}
 
-	if ctx.Config.Source.PrefixTemplate != "" {
-		prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
-		if err != nil {
-			return err
-		}
-		args = append(args, "--prefix", prefix)
+	prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
+	if err != nil {
+		return err
 	}
 
-	files, err := evalFiles(ctx)
+	af, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("could not create archive: %w", err)
+	}
+	defer af.Close() //nolint:errcheck
+
+	arch, err := archive.New(af, ctx.Config.Source.Format)
+	if err != nil {
+		return err
+	}
+
+	var ff []config.File
+	for _, f := range strings.Split(out, "\n") {
+		if strings.TrimSpace(f) == "" {
+			continue
+		}
+		ff = append(ff, config.File{
+			Source: f,
+		})
+	}
+	files, err := archivefiles.Eval(tmpl.New(ctx), append(ff, ctx.Config.Source.Files...))
 	if err != nil {
 		return err
 	}
 	for _, f := range files {
-		args = append(args, "--add-file", f)
+		f.Destination = filepath.Join(prefix, f.Destination)
+		if err := arch.Add(f); err != nil {
+			return fmt.Errorf("could not add %q to archive: %w", f.Source, err)
+		}
 	}
 
-	args = append(args, ctx.Git.FullCommit)
-	out, err := git.Clean(git.Run(ctx, args...))
-	log.Debug(out)
+	if err := arch.Close(); err != nil {
+		return fmt.Errorf("could not close archive file: %w", err)
+	}
+	if err := af.Close(); err != nil {
+		return fmt.Errorf("could not close archive file: %w", err)
+	}
 
 	ctx.Artifacts.Add(&artifact.Artifact{
 		Type: artifact.UploadableSourceArchive,
@@ -70,41 +97,6 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 	return err
 }
 
-// to reuse the archivefiles packages, we do something funky:
-// - convert the []string to []config.File
-// - eval it in archivefiles
-// - convert it back to []string
-//
-// we also handle files already tracked, as if we add them again,
-// they'll get duplicated in the archive.
-func evalFiles(ctx *context.Context) ([]string, error) {
-	var files []config.File
-	for _, f := range ctx.Config.Source.Files {
-		files = append(files, config.File{
-			Source: f,
-		})
-	}
-	addFiles, err := archivefiles.Eval(tmpl.New(ctx), files)
-	if err != nil {
-		return nil, err
-	}
-
-	var result []string
-	for _, f := range addFiles {
-		if isTracked(ctx, f.Source) {
-			continue
-		}
-		result = append(result, f.Source)
-	}
-	return result, nil
-}
-
-// check if file is tracked, and, if it is we should not add it to the archive again.
-func isTracked(ctx *context.Context, path string) bool {
-	_, err := git.Run(ctx, "ls-files", "--error-unmatch", path)
-	return err == nil
-}
-
 // Default sets the pipe defaults.
 func (Pipe) Default(ctx *context.Context) error {
 	archive := &ctx.Config.Source
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 6dcde36c..0bab7252 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -876,11 +876,11 @@ type Publisher struct {
 
 // Source configuration.
 type Source struct {
-	NameTemplate   string   `yaml:"name_template,omitempty" json:"name_template,omitempty"`
-	Format         string   `yaml:"format,omitempty" json:"format,omitempty"`
-	Enabled        bool     `yaml:"enabled,omitempty" json:"enabled,omitempty"`
-	PrefixTemplate string   `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"`
-	Files          []string `yaml:"files,omitempty" json:"files,omitempty"`
+	NameTemplate   string `yaml:"name_template,omitempty" json:"name_template,omitempty"`
+	Format         string `yaml:"format,omitempty" json:"format,omitempty"`
+	Enabled        bool   `yaml:"enabled,omitempty" json:"enabled,omitempty"`
+	PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"`
+	Files          []File `yaml:"files,omitempty" json:"files,omitempty"`
 }
 
 // Project includes all project configuration.
diff --git a/www/docs/customization/source.md b/www/docs/customization/source.md
index ef1f5181..bfec2703 100644
--- a/www/docs/customization/source.md
+++ b/www/docs/customization/source.md
@@ -25,7 +25,6 @@ source:
   prefix_template: '{{ .ProjectName }}-{{ .Version }}/'
 
   # Additional files/template/globs you want to add to the source archive.
-  # Will use --add-file of git-archive.
   # Defaults to empty.
   files:
     - LICENSE.txt
@@ -34,6 +33,22 @@ source:
     - docs/*
     - design/*.png
     - templates/**/*
+    # a more complete example, check the globbing deep dive below
+    - src: '*.md'
+      dst: docs
+      # Strip parent folders when adding files to the archive.
+      # Default: false
+      strip_parent: true
+      # File info.
+      # Not all fields are supported by all formats available formats.
+      # Defaults to the file info of the actual file if not provided.
+      info:
+        owner: root
+        group: root
+        mode: 0644
+        # format is `time.RFC3339Nano`
+        mtime: 2008-01-02T15:04:05Z
+
 ```
 
 !!! tip
diff --git a/www/docs/static/schema.json b/www/docs/static/schema.json
index 4ab5dbdf..24b278aa 100644
--- a/www/docs/static/schema.json
+++ b/www/docs/static/schema.json
@@ -2382,7 +2382,7 @@
 					},
 					"files": {
 						"items": {
-							"type": "string"
+							"$ref": "#/$defs/File"
 						},
 						"type": "array"
 					}


Test patch:
diff --git a/internal/pipe/sourcearchive/source_test.go b/internal/pipe/sourcearchive/source_test.go
index b28926f7..a19b3456 100644
--- a/internal/pipe/sourcearchive/source_test.go
+++ b/internal/pipe/sourcearchive/source_test.go
@@ -27,6 +27,8 @@ func TestArchive(t *testing.T) {
 			testlib.GitCommit(t, "feat: first")
 			require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655))
 			require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655))
+			require.NoError(t, os.MkdirAll("subfolder", 0o755))
+			require.NoError(t, os.WriteFile("subfolder/file.md", []byte("a file within a folder, added later"), 0o655))
 
 			ctx := context.New(config.Project{
 				ProjectName: "foo",
@@ -35,8 +37,9 @@ func TestArchive(t *testing.T) {
 					Format:         format,
 					Enabled:        true,
 					PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/",
-					Files: []string{
-						"*.txt",
+					Files: []config.File{
+						{Source: "*.txt"},
+						{Source: "subfolder/*"},
 					},
 				},
 			})
@@ -65,22 +68,13 @@ func TestArchive(t *testing.T) {
 				return
 			}
 
-			f, err := os.Open(path)
-			require.NoError(t, err)
-			z, err := zip.NewReader(f, stat.Size())
-			require.NoError(t, err)
-
-			var paths []string
-			for _, zf := range z.File {
-				paths = append(paths, zf.Name)
-			}
-			require.Equal(t, []string{
-				"foo-1.0.0/",
+			require.ElementsMatch(t, []string{
 				"foo-1.0.0/README.md",
 				"foo-1.0.0/code.py",
 				"foo-1.0.0/code.txt",
 				"foo-1.0.0/added-later.txt",
-			}, paths)
+				"foo-1.0.0/subfolder/file.md",
+			}, lsZip(t, path))
 		})
 	}
 }
@@ -96,7 +90,7 @@ func TestInvalidFormat(t *testing.T) {
 		},
 	})
 	require.NoError(t, Pipe{}.Default(ctx))
-	require.EqualError(t, Pipe{}.Run(ctx), "fatal: Unknown archive format '7z'")
+	require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z")
 }
 
 func TestDefault(t *testing.T) {
@@ -115,7 +109,45 @@ func TestInvalidNameTemplate(t *testing.T) {
 			NameTemplate: "{{ .foo }-asdda",
 		},
 	})
-	require.EqualError(t, Pipe{}.Run(ctx), "template: tmpl:1: unexpected \"}\" in operand")
+	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
+}
+
+func TestInvalidInvalidFileTemplate(t *testing.T) {
+	testlib.Mktmp(t)
+	require.NoError(t, os.Mkdir("dist", 0o744))
+
+	testlib.GitInit(t)
+	require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655))
+	testlib.GitAdd(t)
+	testlib.GitCommit(t, "feat: first")
+
+	ctx := context.New(config.Project{
+		ProjectName: "foo",
+		Dist:        "dist",
+		Source: config.Source{
+			Format:  "tar.gz",
+			Enabled: true,
+			Files: []config.File{
+				{Source: "{{.Test}"},
+			},
+		},
+	})
+	ctx.Git.FullCommit = "HEAD"
+	ctx.Version = "1.0.0"
+	require.NoError(t, Pipe{}.Default(ctx))
+	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
+}
+
+func TestInvalidPrefixTemplate(t *testing.T) {
+	ctx := context.New(config.Project{
+		Dist: t.TempDir(),
+		Source: config.Source{
+			Enabled:        true,
+			PrefixTemplate: "{{ .ProjectName }/",
+		},
+	})
+	require.NoError(t, Pipe{}.Default(ctx))
+	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
 }
 
 func TestDisabled(t *testing.T) {
@@ -136,3 +168,24 @@ func TestSkip(t *testing.T) {
 		require.False(t, Pipe{}.Skip(ctx))
 	})
 }
+
+func TestString(t *testing.T) {
+	require.NotEmpty(t, Pipe{}.String())
+}
+
+func lsZip(tb testing.TB, path string) []string {
+	tb.Helper()
+
+	stat, err := os.Stat(path)
+	require.NoError(tb, err)
+	f, err := os.Open(path)
+	require.NoError(tb, err)
+	z, err := zip.NewReader(f, stat.Size())
+	require.NoError(tb, err)
+
+	var paths []string
+	for _, zf := range z.File {
+		paths = append(paths, zf.Name)
+	}
+	return paths
+}