Repository: openllb/hlb
Pull request #251 (2021-07-13 22:04:47)

restore doublestar syntax for secret include/excludePatterns
It seems we have lost the ability to use doublestar syntax when
matching secrets for inclusion/exclusion.  Not sure which refactoring
effort lost this, but replacing filepath.Walk with fsutil.Walk seems
to work.  Also adding in some tests to ensure we dont lose this
functionality again.


Patch:
diff --git a/pkg/llbutil/secret.go b/pkg/llbutil/secret.go
index c4c9476..849b9fa 100644
--- a/pkg/llbutil/secret.go
+++ b/pkg/llbutil/secret.go
@@ -1,9 +1,12 @@
 package llbutil
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
+
+	"github.com/tonistiigi/fsutil"
 )
 
 type IncludePatterns struct {
@@ -26,52 +29,16 @@ func FilterLocalFiles(localPath string, includePatterns, excludePatterns []strin
 		localPaths = append(localPaths, localPath)
 		return
 	case fi.Mode().IsDir():
-		err = filepath.Walk(localPath, func(walkPath string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			relPath, err := filepath.Rel(localPath, walkPath)
+		opt := &fsutil.WalkOpt{
+			IncludePatterns: includePatterns,
+			ExcludePatterns: excludePatterns,
+		}
+		err = fsutil.Walk(context.TODO(), localPath, opt, func(walkPath string, info os.FileInfo, err error) error {
 			if err != nil {
 				return err
 			}
-			if relPath == "." {
-				return nil
-			}
-			if len(includePatterns) > 0 {
-				for _, pattern := range includePatterns {
-					if ok, err := filepath.Match(pattern, relPath); ok && err == nil {
-						if info.Mode().IsRegular() {
-							localPaths = append(localPaths, walkPath)
-						}
-						return nil
-					} else if err != nil {
-						return err
-					}
-				}
-				// Didn't match include, so skip directory.
-				if info.Mode().IsDir() {
-					return filepath.SkipDir
-				}
-				return nil
-			} else if len(excludePatterns) > 0 {
-				for _, pattern := range excludePatterns {
-					if ok, err := filepath.Match(pattern, relPath); !ok && err == nil {
-						if info.Mode().IsDir() {
-							return filepath.SkipDir
-						}
-						return nil
-					} else if err != nil {
-						return err
-					}
-				}
-				// Didn't match exclude to add it to list.
-				if info.Mode().IsRegular() {
-					localPaths = append(localPaths, walkPath)
-				}
-				return nil
-			}
 			if info.Mode().IsRegular() {
-				localPaths = append(localPaths, walkPath)
+				localPaths = append(localPaths, filepath.Join(localPath, walkPath))
 			}
 			return nil
 		})


Test patch:
diff --git a/pkg/llbutil/secret_test.go b/pkg/llbutil/secret_test.go
new file mode 100644
index 0000000..b05019c
--- /dev/null
+++ b/pkg/llbutil/secret_test.go
@@ -0,0 +1,85 @@
+package llbutil
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilterLocalFiles(t *testing.T) {
+	localPath, err := os.MkdirTemp("", "test")
+	require.NoError(t, err)
+	files := []string{"decrypted/secret", "other/decrypted/secret", "secret", "src/foo"}
+	for _, f := range files {
+		err = os.MkdirAll(filepath.Dir(filepath.Join(localPath, f)), 0755)
+		require.NoError(t, err)
+		fs, err := os.Create(filepath.Join(localPath, f))
+		require.NoError(t, err)
+		fs.Close()
+	}
+
+	got, err := FilterLocalFiles(localPath, nil, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, files, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"**/nada"}, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Nil(t, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"secret"}, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"secret"}, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"*/secret"}, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"decrypted/secret", "secret"}, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"**/secret"}, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret", "secret"}, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"**/decrypted"}, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret"}, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"**/decrypted"}, []string{"other"})
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"decrypted/secret"}, got)
+
+	got, err = FilterLocalFiles(localPath, []string{"**/secret"}, []string{"secret"})
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret"}, got)
+
+	got, err = FilterLocalFiles(localPath, nil, []string{"secret"})
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"decrypted/secret", "other/decrypted/secret", "src/foo"}, got)
+
+	got, err = FilterLocalFiles(localPath, nil, []string{"**/secret"})
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"src/foo"}, got)
+
+	got, err = FilterLocalFiles(localPath+"/secret", nil, nil)
+	require.NoError(t, err)
+	relativeFiles(localPath, got)
+	require.Equal(t, []string{"secret"}, got)
+}
+
+func relativeFiles(localPath string, localFiles []string) {
+	for i, f := range localFiles {
+		localFiles[i], _ = filepath.Rel(localPath, f)
+	}
+	sort.Strings(localFiles)
+}